Commit ada56835271 for woocommerce
commit ada568352719fe7a06ba3463aec4273538925a29
Author: AnnaMag <AnnaMag@users.noreply.github.com>
Date: Thu Nov 12 15:40:07 2026 +0100
Agent readiness: sitemap check (#70434)
* Agent readiness: sitemap check
Part of #70434.
* Agent readiness: skip off-host sitemaps and do not treat the shop page as a product
A robots.txt Sitemap line on another host was counted as a failed load, so the check stayed at Couldn't check. The shop page URL also matched when the product base is the shop slug.
* Agent readiness: mark the changelog entry as a patch
A Comment-only changelog entry must be a patch; changelogger validate rejects it as minor.
* Agent readiness: count product URLs one level down in a sitemap index
* Agent readiness: open nested sitemap indexes within a request budget, and count a blocked sitemap as couldn't check
Jetpack points /sitemap.xml at sitemap-index-N.xml once a store has more than
1,000 URLs, so a one-level limit missed its products. Child sitemaps are now
opened at any depth, up to five child requests. A sitemap that answers 403 or
5xx is couldn't check rather than a missing sitemap, as robots.txt already is.
Off-host and //host Sitemap lines are dropped before the five-line cap.
diff --git a/plugins/woocommerce/changelog/add-agent-readiness-robots-sitemap b/plugins/woocommerce/changelog/add-agent-readiness-robots-sitemap
new file mode 100644
index 00000000000..9f168a6ae76
--- /dev/null
+++ b/plugins/woocommerce/changelog/add-agent-readiness-robots-sitemap
@@ -0,0 +1,4 @@
+Significance: patch
+Type: dev
+Comment: Agent readiness: a sitemap check, behind an experimental flag; nothing ships to merchants.
+
diff --git a/plugins/woocommerce/src/Internal/AgentReadiness/AgenticCommerceSettingsPage.php b/plugins/woocommerce/src/Internal/AgentReadiness/AgenticCommerceSettingsPage.php
index 4864bc87380..fb9f207f604 100644
--- a/plugins/woocommerce/src/Internal/AgentReadiness/AgenticCommerceSettingsPage.php
+++ b/plugins/woocommerce/src/Internal/AgentReadiness/AgenticCommerceSettingsPage.php
@@ -165,7 +165,7 @@ class AgenticCommerceSettingsPage extends \WC_Settings_Page {
*
* @param array<string, mixed>|null $report Stored report.
* @param array<int, array{id:string}> $live Store conditions read now.
- * @return array<int, array{id:string,level:string,ok:bool|null,name:string,fix_url:string}>
+ * @return array<int, array{id:string,level:string,ok:bool|null,name:string,fix:string,fix_url:string}>
*/
private static function scanned_checks( ?array $report, array $live ): array {
$live_ids = array_column( $live, 'id' );
@@ -179,6 +179,7 @@ class AgenticCommerceSettingsPage extends \WC_Settings_Page {
'level' => StoreConditions::LEVEL_RECOMMENDED,
'ok' => isset( $check['ok'] ) ? (bool) $check['ok'] : null,
'name' => (string) $check['name'],
+ 'fix' => (string) ( $check['fix'] ?? '' ),
'fix_url' => (string) ( $check['fix_url'] ?? '' ),
);
}
@@ -418,8 +419,8 @@ class AgenticCommerceSettingsPage extends \WC_Settings_Page {
* Agent readiness: the checks the last scan ran on what the store returns. They never
* affect the switch, so they sit apart from the setup checks in Status.
*
- * @param array<int, array{id:string,level:string,ok:bool|null,name:string,fix_url:string}> $checks Scanned checks.
- * @param bool $launched Whether the store is launched.
+ * @param array<int, array{id:string,level:string,ok:bool|null,name:string,fix:string,fix_url:string}> $checks Scanned checks.
+ * @param bool $launched Whether the store is launched.
*/
private function render_readiness( array $checks, bool $launched ): void {
?>
@@ -447,7 +448,9 @@ class AgenticCommerceSettingsPage extends \WC_Settings_Page {
<span class="wc-agentic-commerce-check__text"><?php echo esc_html( $check['name'] ); ?></span>
<?php if ( null === $check['ok'] ) : ?>
<span class="description"><?php echo $launched ? esc_html__( 'Your store could not load its own page. The next check tries again.', 'woocommerce' ) : esc_html__( 'Launch your store to run this check.', 'woocommerce' ); ?></span>
- <?php elseif ( ! $check['ok'] && '' !== $check['fix_url'] ) : ?>
+ <?php elseif ( ! $check['ok'] && '' === $check['fix_url'] ) : ?>
+ <span class="description"><?php echo esc_html( $check['fix'] ); ?></span>
+ <?php elseif ( ! $check['ok'] ) : ?>
<a class="button" href="<?php echo esc_url( $check['fix_url'] ); ?>"><?php echo str_starts_with( $check['fix_url'], admin_url() ) ? esc_html__( 'Open settings', 'woocommerce' ) : esc_html__( 'Learn how', 'woocommerce' ); ?></a>
<?php endif; ?>
</li>
diff --git a/plugins/woocommerce/src/Internal/AgentReadiness/README.md b/plugins/woocommerce/src/Internal/AgentReadiness/README.md
index 393227a0deb..9d638d99378 100644
--- a/plugins/woocommerce/src/Internal/AgentReadiness/README.md
+++ b/plugins/woocommerce/src/Internal/AgentReadiness/README.md
@@ -55,6 +55,12 @@ Store-wide conditions (`StoreConditions`) are reported first because they affect
`StoreOutputChecks` runs in the background scan only. It loads the store's own pages as a logged-out visitor (`StoreResponses`): only URLs on the store's host, redirects followed on that host only, a 5 second timeout and 512 KB per response, each URL once per scan, nothing at all in coming soon mode. Its results go into the report's `store_checks` with the same keys, with `ok` set to `null` when the store couldn't load its own page ("Couldn't check"). They are always `recommended`, so they never block the UCP switch.
+| Id | Passes when |
+| --- | --- |
+| `sitemap_lists_products` | A sitemap listed in robots.txt, or `/wp-sitemap.xml`, `/sitemap_index.xml` or `/sitemap.xml`, names a product sitemap or product pages. An index that does not has its child sitemaps opened, at any depth, up to five child requests. |
+
+robots.txt is loaded from the host root, only to find the sitemaps it lists. Its crawler rules aren't checked: a host or CDN can rewrite the file or block crawlers without it, and the store's request to itself skips both, so they wait for an outside view of the store (WOOPRD-3953). The sitemap check is "couldn't check" when robots.txt could not be read, or a candidate sitemap could not be loaded or answered other than 200, 404 or 410, and no fetched sitemap names products. A category sitemap does not count. When an index does not name products, its child sitemaps on this host are checked, nested indexes included (Jetpack serves one past 1,000 URLs), up to five child requests in all. Sitemap lines on another host, or written as `//host/path`, are skipped and do not use up the five listed slots.
+
## Report format
`AgentReadiness::get_report()` returns:
diff --git a/plugins/woocommerce/src/Internal/AgentReadiness/RobotsTxt.php b/plugins/woocommerce/src/Internal/AgentReadiness/RobotsTxt.php
new file mode 100644
index 00000000000..177ae003489
--- /dev/null
+++ b/plugins/woocommerce/src/Internal/AgentReadiness/RobotsTxt.php
@@ -0,0 +1,46 @@
+<?php
+declare( strict_types = 1 );
+
+namespace Automattic\WooCommerce\Internal\AgentReadiness;
+
+defined( 'ABSPATH' ) || exit;
+
+/**
+ * Reads the Sitemap lines of a robots.txt. Its crawler rules aren't judged from inside the store:
+ * a host or CDN can rewrite the file, and a request from the store to itself skips them.
+ *
+ * @since 11.4.0
+ */
+class RobotsTxt {
+
+ /**
+ * Sitemap URLs the file lists.
+ *
+ * @var array<int, string>
+ */
+ private array $sitemaps = array();
+
+ /**
+ * Parse a robots.txt.
+ *
+ * @param string $text File contents.
+ */
+ public function __construct( string $text ) {
+ $lines = preg_split( '/\r\n|\r|\n/', $text );
+ foreach ( false === $lines ? array() : $lines as $line ) {
+ $parts = explode( ':', (string) preg_replace( '/#.*/', '', $line ), 2 );
+ if ( 2 === count( $parts ) && 'sitemap' === strtolower( trim( $parts[0] ) ) && '' !== trim( $parts[1] ) ) {
+ $this->sitemaps[] = trim( $parts[1] );
+ }
+ }
+ }
+
+ /**
+ * Sitemap URLs listed in the file.
+ *
+ * @return array<int, string>
+ */
+ public function sitemaps(): array {
+ return $this->sitemaps;
+ }
+}
diff --git a/plugins/woocommerce/src/Internal/AgentReadiness/StoreOutputChecks.php b/plugins/woocommerce/src/Internal/AgentReadiness/StoreOutputChecks.php
index 52c07d81990..e8863747197 100644
--- a/plugins/woocommerce/src/Internal/AgentReadiness/StoreOutputChecks.php
+++ b/plugins/woocommerce/src/Internal/AgentReadiness/StoreOutputChecks.php
@@ -73,12 +73,173 @@ class StoreOutputChecks {
return $results;
}
+ private const MAX_LISTED_SITEMAPS = 5;
+
+ private const MAX_CHILD_SITEMAPS = 5;
+
+ /**
+ * Statuses that answer whether a file exists. Anything else (a firewall's 403, a 5xx) is couldn't check.
+ */
+ private const ANSWER_STATUSES = array( 200, 404, 410 );
+
/**
* The checks, each with a test that returns true, false, or null for couldn't check.
*
* @return array<int, array{id:string,test:callable(StoreResponses):(bool|null),blocks:string,name:string,label:string,why:string,fix:string,fix_url:string}>
*/
protected function checks(): array {
- return array();
+ return array(
+ array(
+ 'id' => 'sitemap_lists_products',
+ 'test' => fn( StoreResponses $responses ): ?bool => $this->sitemap_lists_products( $responses ),
+ 'blocks' => 'listing',
+ 'name' => __( 'A sitemap lists your products', 'woocommerce' ),
+ 'label' => __( 'No sitemap lists your products', 'woocommerce' ),
+ 'why' => __( 'Crawlers use the sitemap to find new and changed products.', 'woocommerce' ),
+ 'fix' => __( 'Turn on the sitemap in your SEO plugin, or remove the code that turns off the WordPress sitemap, and include products in it.', 'woocommerce' ),
+ 'fix_url' => '',
+ ),
+ );
+ }
+
+ /**
+ * The robots.txt crawlers get. '' when the store answered that there is none.
+ * Null when it could not be read: the request failed, or the body is a page.
+ *
+ * Loaded from the host root, also when WordPress is in a subdirectory.
+ *
+ * @param StoreResponses $responses Store responses.
+ */
+ private function robots_txt( StoreResponses $responses ): ?string {
+ $home = (array) wp_parse_url( home_url() );
+ $response = $responses->get( ( $home['scheme'] ?? 'http' ) . '://' . ( $home['host'] ?? '' ) . ( isset( $home['port'] ) ? ':' . $home['port'] : '' ) . '/robots.txt' );
+ if ( null === $response || ! in_array( $response['status'], self::ANSWER_STATUSES, true ) ) {
+ return null;
+ }
+ if ( in_array( $response['status'], array( 404, 410 ), true ) ) {
+ return '';
+ }
+ // A theme or a login wall that answers every URL with a page is not a robots.txt.
+ return str_starts_with( ltrim( $response['body'] ), '<' ) ? null : $response['body'];
+ }
+
+ /**
+ * Whether a sitemap the store serves names products. An index that does not
+ * has its child sitemaps on this host opened, at any depth, up to five child
+ * requests in all. Null when robots.txt or one of those sitemaps could not
+ * be loaded and none named products.
+ *
+ * @param StoreResponses $responses Store responses.
+ */
+ private function sitemap_lists_products( StoreResponses $responses ): ?bool {
+ $robots = $this->robots_txt( $responses );
+ $listed = null === $robots ? array() : array_slice(
+ array_values( array_filter( ( new RobotsTxt( $robots ) )->sitemaps(), array( StoreResponses::class, 'is_store_url' ) ) ),
+ 0,
+ self::MAX_LISTED_SITEMAPS
+ );
+ $defaults = array( '/sitemap_index.xml', '/sitemap.xml' );
+ if ( function_exists( 'wp_sitemaps_get_server' ) && wp_sitemaps_get_server()->sitemaps_enabled() ) {
+ $defaults[] = '/wp-sitemap.xml';
+ }
+ $queue = array_values( array_unique( array_map( array( self::class, 'absolute_url' ), array_merge( $listed, $defaults ) ) ) );
+ $seen = array_fill_keys( $queue, true );
+ $budget = count( $queue ) + self::MAX_CHILD_SITEMAPS;
+ $unknown = null === $robots;
+ for ( $i = 0; isset( $queue[ $i ] ) && $i < $budget; $i++ ) {
+ $body = $this->sitemap_body( $responses, $queue[ $i ], $unknown );
+ if ( null === $body ) {
+ continue;
+ }
+ if ( self::names_products( $body ) ) {
+ return true;
+ }
+ foreach ( self::index_child_urls( $body ) as $child ) {
+ if ( ! StoreResponses::is_store_url( $child ) ) {
+ continue;
+ }
+ $child = self::absolute_url( $child );
+ if ( ! isset( $seen[ $child ] ) ) {
+ $seen[ $child ] = true;
+ $queue[] = $child;
+ }
+ }
+ }
+ return $unknown ? null : false;
+ }
+
+ /**
+ * A path from the home URL as a full URL, so one sitemap is one queue entry however it was written.
+ *
+ * @param string $url URL or path.
+ */
+ private static function absolute_url( string $url ): string {
+ return str_starts_with( $url, '/' ) ? home_url( $url ) : $url;
+ }
+
+ /**
+ * Body of a sitemap response with status 200, or null. A failed request, or
+ * any status other than 200, 404 or 410 (a firewall's 403, a 5xx), marks the check unknown.
+ *
+ * @param StoreResponses $responses Store responses.
+ * @param string $url URL or path.
+ * @param bool $unknown Whether a sitemap could not be loaded. Passed by reference.
+ * @return string|null
+ */
+ private function sitemap_body( StoreResponses $responses, string $url, bool &$unknown ): ?string {
+ $response = $responses->get( $url );
+ if ( null === $response || ! in_array( $response['status'], self::ANSWER_STATUSES, true ) ) {
+ $unknown = true;
+ return null;
+ }
+ return 200 === $response['status'] ? $response['body'] : null;
+ }
+
+ /**
+ * Child sitemap URLs listed in a sitemap index. Empty when the body is not an index.
+ *
+ * @param string $xml Sitemap body.
+ * @return array<int, string>
+ */
+ private static function index_child_urls( string $xml ): array {
+ if ( false === stripos( $xml, '<sitemapindex' ) ) {
+ return array();
+ }
+ $urls = array();
+ $offset = 0;
+ $start = stripos( $xml, '<loc>', $offset );
+ while ( false !== $start ) {
+ $start += strlen( '<loc>' );
+ $end = stripos( $xml, '</loc>', $start );
+ if ( false === $end ) {
+ break;
+ }
+ $url = trim( str_replace( array( '<![CDATA[', ']]>' ), '', substr( $xml, $start, $end - $start ) ) );
+ if ( '' !== $url ) {
+ $urls[] = $url;
+ }
+ $offset = $end + strlen( '</loc>' );
+ $start = stripos( $xml, '<loc>', $offset );
+ }
+ return $urls;
+ }
+
+ /**
+ * Whether a sitemap index names a product sitemap (product-sitemap.xml,
+ * wp-sitemap-posts-product-1.xml), or a urlset lists a product page. A category
+ * sitemap and a page that isn't a sitemap never count.
+ *
+ * @param string $xml Sitemap body.
+ */
+ private static function names_products( string $xml ): bool {
+ if ( ! preg_match( '#<(urlset|sitemapindex)[\s>]#i', $xml ) ) {
+ return false;
+ }
+ if ( 1 === preg_match( '#<loc>[^<]*(?:product-sitemap|wp-sitemap-posts-product-)[^<]*\.xml\s*</loc>#i', $xml ) ) {
+ return true;
+ }
+ // The fixed part of the product base, before a tag such as %product_cat%.
+ $base = trim( (string) strtok( (string) ( wc_get_permalink_structure()['product_base'] ?? '' ), '%' ), '/' );
+ return '' !== $base && 1 === preg_match( '#<loc>[^<]*/' . preg_quote( $base, '#' ) . '/[^<]+</loc>#i', $xml );
}
}
diff --git a/plugins/woocommerce/src/Internal/AgentReadiness/StoreResponses.php b/plugins/woocommerce/src/Internal/AgentReadiness/StoreResponses.php
index 29cb81e5267..aa6caff149b 100644
--- a/plugins/woocommerce/src/Internal/AgentReadiness/StoreResponses.php
+++ b/plugins/woocommerce/src/Internal/AgentReadiness/StoreResponses.php
@@ -101,6 +101,22 @@ class StoreResponses {
return null;
}
+ /**
+ * Whether a path or URL is on the store's host. Paths from the home URL are;
+ * a //host/path reference is not, since a sitemap location must be a full URL.
+ * Used so a robots.txt Sitemap line on another host is skipped, not treated
+ * as a failed load of the store's own pages.
+ *
+ * @param string $url URL or path.
+ */
+ public static function is_store_url( string $url ): bool {
+ if ( str_starts_with( $url, '/' ) ) {
+ return ! str_starts_with( $url, '//' );
+ }
+ $store = self::origin( home_url() );
+ return null !== $store && self::origin( $url ) === $store;
+ }
+
/**
* Host and port of an http(s) URL, or null for any other scheme. The port is
* only the one written in the URL, so moving from http to https on the
diff --git a/plugins/woocommerce/tests/php/src/Internal/AgentReadiness/AgentReadinessTest.php b/plugins/woocommerce/tests/php/src/Internal/AgentReadiness/AgentReadinessTest.php
index 2ed7344ada6..879e7b73111 100644
--- a/plugins/woocommerce/tests/php/src/Internal/AgentReadiness/AgentReadinessTest.php
+++ b/plugins/woocommerce/tests/php/src/Internal/AgentReadiness/AgentReadinessTest.php
@@ -30,6 +30,8 @@ class AgentReadinessTest extends WC_Unit_Test_Case {
*/
public function setUp(): void {
parent::setUp();
+ // Report tests stay offline: store output checks would load the test site's pages.
+ StoreOutputChecksTest::block_requests();
$store_conditions = new StoreConditions();
$store_conditions->init( new ComingSoonHelper() );
$output_checks = new StoreOutputChecks();
diff --git a/plugins/woocommerce/tests/php/src/Internal/AgentReadiness/AgenticCommerceSettingsTest.php b/plugins/woocommerce/tests/php/src/Internal/AgentReadiness/AgenticCommerceSettingsTest.php
index 7accf743266..a150c467a8c 100644
--- a/plugins/woocommerce/tests/php/src/Internal/AgentReadiness/AgenticCommerceSettingsTest.php
+++ b/plugins/woocommerce/tests/php/src/Internal/AgentReadiness/AgenticCommerceSettingsTest.php
@@ -27,6 +27,8 @@ class AgenticCommerceSettingsTest extends WC_Unit_Test_Case {
*/
public function setUp(): void {
parent::setUp();
+ // Report tests stay offline: store output checks would load the test site's pages.
+ StoreOutputChecksTest::block_requests();
include_once WC_ABSPATH . 'includes/admin/class-wc-admin-settings.php';
include_once WC_ABSPATH . 'includes/admin/settings/class-wc-settings-page.php';
set_current_screen( 'dashboard' );
diff --git a/plugins/woocommerce/tests/php/src/Internal/AgentReadiness/ReadinessRestControllerTest.php b/plugins/woocommerce/tests/php/src/Internal/AgentReadiness/ReadinessRestControllerTest.php
index 41b91a0baea..2e0ca7343de 100644
--- a/plugins/woocommerce/tests/php/src/Internal/AgentReadiness/ReadinessRestControllerTest.php
+++ b/plugins/woocommerce/tests/php/src/Internal/AgentReadiness/ReadinessRestControllerTest.php
@@ -21,6 +21,8 @@ class ReadinessRestControllerTest extends WC_REST_Unit_Test_Case {
public function setUp(): void {
parent::setUp();
add_filter( 'pre_option_woocommerce_feature_agent_readiness_enabled', array( __CLASS__, 'enabled' ) );
+ // Report tests stay offline: store output checks would load the test site's pages.
+ StoreOutputChecksTest::block_requests();
delete_option( AgentReadiness::REPORT_OPTION );
as_unschedule_all_actions( ScheduledScan::RUN_NOW_HOOK );
wp_set_current_user( self::factory()->user->create( array( 'role' => 'administrator' ) ) );
diff --git a/plugins/woocommerce/tests/php/src/Internal/AgentReadiness/RobotsTxtTest.php b/plugins/woocommerce/tests/php/src/Internal/AgentReadiness/RobotsTxtTest.php
new file mode 100644
index 00000000000..f4e8bdd3115
--- /dev/null
+++ b/plugins/woocommerce/tests/php/src/Internal/AgentReadiness/RobotsTxtTest.php
@@ -0,0 +1,24 @@
+<?php
+declare( strict_types = 1 );
+
+namespace Automattic\WooCommerce\Tests\Internal\AgentReadiness;
+
+use Automattic\WooCommerce\Internal\AgentReadiness\RobotsTxt;
+use WC_Unit_Test_Case;
+
+/**
+ * Tests for reading the sitemaps a robots.txt lists.
+ */
+class RobotsTxtTest extends WC_Unit_Test_Case {
+
+ /**
+ * @testdox Sitemap lines are listed wherever they appear, in any case and line ending; comments and empty values are not.
+ */
+ public function test_sitemaps(): void {
+ $this->assertSame( array(), ( new RobotsTxt( '' ) )->sitemaps() );
+ $this->assertSame(
+ array( 'https://example.com/sitemap_index.xml', 'https://example.com/products.xml' ),
+ ( new RobotsTxt( "Sitemap: https://example.com/sitemap_index.xml\r\nUser-agent: *\r\nDisallow:\r\n# Sitemap: https://example.com/old.xml\nSITEMAP: https://example.com/products.xml\nSitemap:\n" ) )->sitemaps()
+ );
+ }
+}
diff --git a/plugins/woocommerce/tests/php/src/Internal/AgentReadiness/ScheduledScanTest.php b/plugins/woocommerce/tests/php/src/Internal/AgentReadiness/ScheduledScanTest.php
index afb44b7ca1d..45885391e29 100644
--- a/plugins/woocommerce/tests/php/src/Internal/AgentReadiness/ScheduledScanTest.php
+++ b/plugins/woocommerce/tests/php/src/Internal/AgentReadiness/ScheduledScanTest.php
@@ -26,6 +26,8 @@ class ScheduledScanTest extends WC_Unit_Test_Case {
*/
public function setUp(): void {
parent::setUp();
+ // Report tests stay offline: store output checks would load the test site's pages.
+ StoreOutputChecksTest::block_requests();
update_option( 'woocommerce_feature_agent_readiness_enabled', 'yes' );
add_filter( 'woocommerce_agent_readiness_rules_url', '__return_empty_string' );
delete_option( AgentReadiness::REPORT_OPTION );
diff --git a/plugins/woocommerce/tests/php/src/Internal/AgentReadiness/StatusTabTest.php b/plugins/woocommerce/tests/php/src/Internal/AgentReadiness/StatusTabTest.php
index 3441897fbb9..de6b319f6d6 100644
--- a/plugins/woocommerce/tests/php/src/Internal/AgentReadiness/StatusTabTest.php
+++ b/plugins/woocommerce/tests/php/src/Internal/AgentReadiness/StatusTabTest.php
@@ -21,6 +21,7 @@ class StatusTabTest extends WC_Unit_Test_Case {
wp_set_current_user( self::factory()->user->create( array( 'role' => 'administrator' ) ) );
$product = WC_Helper_Product::create_simple_product();
update_option( 'blog_public', '0' );
+ StoreOutputChecksTest::block_requests();
wc_get_container()->get( AgentReadiness::class )->refresh_report();
ob_start();
@@ -42,6 +43,7 @@ class StatusTabTest extends WC_Unit_Test_Case {
$hidden->set_catalog_visibility( 'hidden' );
$hidden->save();
WC_Helper_Product::create_external_product();
+ StoreOutputChecksTest::block_requests();
wc_get_container()->get( AgentReadiness::class )->refresh_report();
ob_start();
diff --git a/plugins/woocommerce/tests/php/src/Internal/AgentReadiness/StoreOutputChecksTest.php b/plugins/woocommerce/tests/php/src/Internal/AgentReadiness/StoreOutputChecksTest.php
new file mode 100644
index 00000000000..0513eb07957
--- /dev/null
+++ b/plugins/woocommerce/tests/php/src/Internal/AgentReadiness/StoreOutputChecksTest.php
@@ -0,0 +1,343 @@
+<?php
+declare( strict_types = 1 );
+
+namespace Automattic\WooCommerce\Tests\Internal\AgentReadiness;
+
+use Automattic\WooCommerce\Internal\AgentReadiness\StoreOutputChecks;
+use WC_Unit_Test_Case;
+
+/**
+ * Tests for the robots.txt and sitemap store checks.
+ */
+class StoreOutputChecksTest extends WC_Unit_Test_Case {
+
+ /**
+ * The System Under Test.
+ *
+ * @var StoreOutputChecks
+ */
+ private $sut;
+
+ /**
+ * Pretty permalinks, as on a live store; no HTTP response unless a test serves one.
+ */
+ public function setUp(): void {
+ parent::setUp();
+ $this->set_permalink_structure( '/%postname%/' );
+ $this->sut = wc_get_container()->get( StoreOutputChecks::class );
+ }
+
+ /**
+ * Restore plain permalinks, which live on the global rewrite object outside the rollback.
+ */
+ public function tearDown(): void {
+ try {
+ $this->set_permalink_structure( '' );
+ } finally {
+ parent::tearDown();
+ }
+ }
+
+ /**
+ * Fail every HTTP request, as if the store couldn't reach itself.
+ */
+ public static function block_requests(): void {
+ add_filter( 'pre_http_request', static fn() => new \WP_Error( 'http_request_failed', 'Blocked in tests' ) );
+ }
+
+ /**
+ * Answer requests by path from the home URL; anything else fails like a timeout.
+ *
+ * @param array<string, array{0:int,1:string}> $responses Status and body by path.
+ */
+ private function serve( array $responses ): void {
+ add_filter(
+ 'pre_http_request',
+ static function ( $pre, $args, $url ) use ( $responses ) {
+ unset( $pre, $args );
+ $path = substr( $url, strlen( untrailingslashit( home_url() ) ) );
+ if ( ! isset( $responses[ $path ] ) ) {
+ return new \WP_Error( 'http_request_failed', 'Timed out' );
+ }
+ return array(
+ 'headers' => array(),
+ 'body' => $responses[ $path ][1],
+ 'response' => array(
+ 'code' => $responses[ $path ][0],
+ 'message' => '',
+ ),
+ 'cookies' => array(),
+ );
+ },
+ 10,
+ 3
+ );
+ }
+
+ /**
+ * Results by check id.
+ *
+ * @return array<string, bool|null>
+ */
+ private function results(): array {
+ return array_column( $this->sut->all(), 'ok', 'id' );
+ }
+
+ /**
+ * @testdox When the store cannot load its own pages, the sitemap check is couldn't check.
+ */
+ public function test_unreachable_store_couldnt_check(): void {
+ $this->serve( array() );
+
+ $this->assertNull( $this->results()['sitemap_lists_products'] );
+ }
+
+ /**
+ * @testdox When robots.txt is a page rather than a file, the sitemap check is couldn't check unless a default sitemap lists products.
+ */
+ public function test_robots_txt_page(): void {
+ add_filter( 'wp_sitemaps_enabled', '__return_false' );
+ $this->serve( array( '/robots.txt' => array( 200, '<!doctype html><title>Shop</title>' ) ) );
+ $this->assertNull( $this->results()['sitemap_lists_products'] );
+
+ remove_all_filters( 'pre_http_request' );
+ $this->serve(
+ array(
+ '/robots.txt' => array( 200, '<!doctype html><title>Shop</title>' ),
+ '/sitemap.xml' => array( 200, '<sitemapindex><sitemap><loc>' . home_url( '/product-sitemap.xml' ) . '</loc></sitemap></sitemapindex>' ),
+ )
+ );
+ $this->assertTrue( $this->results()['sitemap_lists_products'] );
+ }
+
+ /**
+ * @testdox With the WordPress sitemap off, a product sitemap listed in robots.txt or at /sitemap_index.xml passes.
+ */
+ public function test_seo_plugin_sitemap(): void {
+ add_filter( 'wp_sitemaps_enabled', '__return_false' );
+ $this->serve(
+ array(
+ '/robots.txt' => array( 200, "User-agent: *\nDisallow:\nSitemap: " . home_url( '/seo/sitemap_index.xml' ) . "\n" ),
+ '/seo/sitemap_index.xml' => array( 200, '<sitemapindex><sitemap><loc>' . home_url( '/product-sitemap.xml' ) . '</loc></sitemap></sitemapindex>' ),
+ )
+ );
+
+ $this->assertTrue( $this->results()['sitemap_lists_products'] );
+ }
+
+ /**
+ * @testdox Only the first five sitemaps robots.txt lists are loaded.
+ */
+ public function test_listed_sitemaps_capped(): void {
+ add_filter( 'wp_sitemaps_enabled', '__return_false' );
+ $requested = array();
+ $robots = "User-agent: *\nDisallow:\n";
+ for ( $i = 1; $i <= 50; $i++ ) {
+ $robots .= 'Sitemap: ' . home_url( "/s$i.xml" ) . "\n";
+ }
+ $this->serve( array( '/robots.txt' => array( 200, $robots ) ) );
+ add_filter(
+ 'pre_http_request',
+ static function ( $pre, $args, $url ) use ( &$requested ) {
+ unset( $args );
+ $requested[] = $url;
+ return $pre;
+ },
+ 5,
+ 3
+ );
+
+ $this->results();
+
+ $this->assertCount( 8, array_unique( $requested ), 'robots.txt, five listed sitemaps, /sitemap_index.xml and /sitemap.xml' );
+ }
+
+ /**
+ * @testdox Jetpack's sitemap index passes when the child file lists a product, and a child with no product does not.
+ */
+ public function test_jetpack_sitemap_index(): void {
+ add_filter( 'wp_sitemaps_enabled', '__return_false' );
+ $index = '<sitemapindex><sitemap><loc>' . home_url( '/sitemap-1.xml' ) . '</loc></sitemap></sitemapindex>';
+ $this->serve(
+ array(
+ '/robots.txt' => array( 200, "User-agent: *\nDisallow:\nSitemap: " . home_url( '/sitemap.xml' ) . "\n" ),
+ '/sitemap.xml' => array( 200, $index ),
+ '/sitemap-1.xml' => array( 200, '<urlset><url><loc>' . home_url( '/product/beanie/' ) . '</loc></url></urlset>' ),
+ )
+ );
+ $this->assertTrue( $this->results()['sitemap_lists_products'] );
+
+ remove_all_filters( 'pre_http_request' );
+ $this->serve(
+ array(
+ '/robots.txt' => array( 200, "User-agent: *\nDisallow:\nSitemap: " . home_url( '/sitemap.xml' ) . "\n" ),
+ '/sitemap.xml' => array( 200, '<sitemapindex><sitemap><loc>https://cdn.example/sitemap.xml</loc></sitemap><sitemap><loc>' . home_url( '/sitemap-1.xml' ) . '</loc></sitemap></sitemapindex>' ),
+ '/sitemap-1.xml' => array( 200, '<urlset><url><loc>' . home_url( '/about/' ) . '</loc></url></urlset>' ),
+ '/sitemap_index.xml' => array( 404, '' ),
+ )
+ );
+ $this->assertFalse( $this->results()['sitemap_lists_products'] );
+ }
+
+ /**
+ * @testdox A nested index is opened, as Jetpack serves past 1,000 URLs, and only five child sitemaps are loaded in all.
+ */
+ public function test_child_sitemaps_capped_by_requests(): void {
+ add_filter( 'wp_sitemaps_enabled', '__return_false' );
+ $product = '<urlset><url><loc>' . home_url( '/product/beanie/' ) . '</loc></url></urlset>';
+ $this->serve(
+ array(
+ '/robots.txt' => array( 404, '' ),
+ '/sitemap.xml' => array( 200, '<sitemapindex><sitemap><loc>' . home_url( '/sitemap-index-1.xml' ) . '</loc></sitemap></sitemapindex>' ),
+ '/sitemap-index-1.xml' => array( 200, '<sitemapindex><sitemap><loc>' . home_url( '/sitemap-1.xml' ) . '</loc></sitemap></sitemapindex>' ),
+ '/sitemap-1.xml' => array( 200, $product ),
+ '/sitemap_index.xml' => array( 404, '' ),
+ )
+ );
+ $this->assertTrue( $this->results()['sitemap_lists_products'], 'A nested index is opened' );
+
+ remove_all_filters( 'pre_http_request' );
+ $locs = '';
+ $about = '<urlset><url><loc>' . home_url( '/about/' ) . '</loc></url></urlset>';
+ $served = array(
+ '/robots.txt' => array( 404, '' ),
+ '/sitemap_index.xml' => array( 404, '' ),
+ );
+ for ( $i = 1; $i <= 6; $i++ ) {
+ $path = '/child-' . $i . '.xml';
+ $locs .= '<sitemap><loc>' . home_url( $path ) . '</loc></sitemap>';
+ $served[ $path ] = array( 200, 6 === $i ? $product : $about );
+ }
+ $served['/sitemap.xml'] = array( 200, '<sitemapindex>' . $locs . '</sitemapindex>' );
+ $requested = array();
+ $this->serve( $served );
+ add_filter(
+ 'pre_http_request',
+ static function ( $pre, $args, $url ) use ( &$requested ) {
+ unset( $args );
+ $requested[] = $url;
+ return $pre;
+ },
+ 5,
+ 3
+ );
+
+ $this->assertFalse( $this->results()['sitemap_lists_products'] );
+ $this->assertContains( home_url( '/child-5.xml' ), $requested, 'Five child sitemaps are opened' );
+ $this->assertNotContains( home_url( '/child-6.xml' ), $requested, 'The sixth is past the cap' );
+ }
+
+ /**
+ * @testdox A sitemap robots.txt lists on another host is skipped, not couldn't check.
+ */
+ public function test_off_host_listed_sitemap_is_skipped(): void {
+ add_filter( 'wp_sitemaps_enabled', '__return_false' );
+ $this->serve(
+ array(
+ '/robots.txt' => array( 200, "User-agent: *\nDisallow:\nSitemap: https://cdn.example/sitemap.xml\n" ),
+ '/sitemap_index.xml' => array( 404, '' ),
+ '/sitemap.xml' => array( 404, '' ),
+ )
+ );
+
+ $this->assertFalse( $this->results()['sitemap_lists_products'] );
+ }
+
+ /**
+ * @testdox Sitemaps robots.txt lists on another host, or as //host references, do not use up the five slots.
+ */
+ public function test_off_host_listed_sitemaps_do_not_use_the_cap(): void {
+ add_filter( 'wp_sitemaps_enabled', '__return_false' );
+ $robots = "User-agent: *\nDisallow:\n";
+ for ( $i = 1; $i <= 5; $i++ ) {
+ $robots .= ( 1 === $i % 2 ? "Sitemap: https://cdn.example/s$i.xml\n" : "Sitemap: //cdn.example/s$i.xml\n" );
+ }
+ $robots .= 'Sitemap: ' . home_url( '/seo/product-list.xml' ) . "\n";
+ $this->serve(
+ array(
+ '/robots.txt' => array( 200, $robots ),
+ '/seo/product-list.xml' => array( 200, '<sitemapindex><sitemap><loc>' . home_url( '/product-sitemap.xml' ) . '</loc></sitemap></sitemapindex>' ),
+ '/sitemap_index.xml' => array( 404, '' ),
+ '/sitemap.xml' => array( 404, '' ),
+ )
+ );
+
+ $this->assertTrue( $this->results()['sitemap_lists_products'] );
+ }
+
+ /**
+ * @testdox A sitemap that answers 403 or 5xx is couldn't check, not a missing sitemap.
+ */
+ public function test_blocked_sitemap_is_unknown(): void {
+ add_filter( 'wp_sitemaps_enabled', '__return_false' );
+ foreach ( array( 403, 503 ) as $status ) {
+ remove_all_filters( 'pre_http_request' );
+ $this->serve(
+ array(
+ '/robots.txt' => array( 404, '' ),
+ '/sitemap_index.xml' => array( 404, '' ),
+ '/sitemap.xml' => array( $status, '<html>Blocked</html>' ),
+ )
+ );
+ $this->assertNull( $this->results()['sitemap_lists_products'], "Status $status" );
+ }
+ }
+
+ /**
+ * @testdox The shop page URL is not a product page when the product base is the shop slug.
+ */
+ public function test_shop_page_url_is_not_a_product_in_the_sitemap(): void {
+ $names = new \ReflectionMethod( StoreOutputChecks::class, 'names_products' );
+ $names->setAccessible( true );
+ add_filter(
+ 'pre_option_woocommerce_permalinks',
+ static fn() => array( 'product_base' => 'shop' )
+ );
+ $shop = '<urlset><url><loc>' . home_url( '/shop/' ) . '</loc></url></urlset>';
+ $item = '<urlset><url><loc>' . home_url( '/shop/beanie/' ) . '</loc></url></urlset>';
+
+ $this->assertFalse( $names->invoke( null, $shop ) );
+ $this->assertTrue( $names->invoke( null, $item ) );
+ }
+
+ /**
+ * @testdox With the WordPress sitemap off, no sitemap is missing, and an unreachable one is couldn't check.
+ */
+ public function test_sitemap_off(): void {
+ add_filter( 'wp_sitemaps_enabled', '__return_false' );
+ $this->serve(
+ array(
+ '/robots.txt' => array( 404, '' ),
+ '/sitemap_index.xml' => array( 404, '' ),
+ '/sitemap.xml' => array( 200, '<urlset><url><loc>' . home_url( '/about/' ) . '</loc></url></urlset>' ),
+ )
+ );
+ $this->assertFalse( $this->results()['sitemap_lists_products'] );
+
+ remove_all_filters( 'pre_http_request' );
+ $this->serve(
+ array(
+ '/robots.txt' => array( 404, '' ),
+ '/sitemap_index.xml' => array( 200, '<!doctype html><a href="' . home_url( '/product/beanie/' ) . '">Beanie</a>' ),
+ '/sitemap.xml' => array( 404, '' ),
+ )
+ );
+ $this->assertFalse( $this->results()['sitemap_lists_products'], 'A page that is not a sitemap does not count' );
+
+ remove_all_filters( 'pre_http_request' );
+ $category_sitemap = home_url( '/wp-sitemap-taxonomies-product_cat-1.xml' );
+ $this->serve(
+ array(
+ '/robots.txt' => array( 200, "User-agent: *\nDisallow:\nSitemap: $category_sitemap\n" ),
+ '/wp-sitemap-taxonomies-product_cat-1.xml' => array( 200, '<sitemapindex><sitemap><loc>' . $category_sitemap . '</loc></sitemap></sitemapindex>' ),
+ '/sitemap_index.xml' => array( 404, '' ),
+ '/sitemap.xml' => array( 404, '' ),
+ )
+ );
+ $this->assertFalse( $this->results()['sitemap_lists_products'], 'A product category sitemap does not list products' );
+
+ remove_all_filters( 'pre_http_request' );
+ $this->serve( array() );
+ $this->assertNull( $this->results()['sitemap_lists_products'] );
+ }
+}