Commit a418b316383 for woocommerce
commit a418b316383eb7f1986372c0cff69fab7248a7ff
Author: Michal Iwanow <4765119+mcliwanow@users.noreply.github.com>
Date: Tue Oct 6 14:21:50 2026 +0200
Add an A/B test for the Marketplace task in Things to do next (#69283)
* Add an A/B test for the Marketplace task title and position
* Add changelog entry for the Marketplace task A/B test
* Move the Marketplace task experiment end date check into a helper
* Respect tracking opt-out filters in the Marketplace task experiment
* Make the Marketplace task A/B test changelog entry merchant-facing
diff --git a/plugins/woocommerce/changelog/wccom-2946-ab-test-marketplace-task b/plugins/woocommerce/changelog/wccom-2946-ab-test-marketplace-task
new file mode 100644
index 00000000000..06470c77f44
--- /dev/null
+++ b/plugins/woocommerce/changelog/wccom-2946-ab-test-marketplace-task
@@ -0,0 +1,4 @@
+Significance: patch
+Type: tweak
+
+Test new wording and placement for the Marketplace task in "Things to do next", for stores in an A/B test.
diff --git a/plugins/woocommerce/src/Admin/API/OnboardingTasks.php b/plugins/woocommerce/src/Admin/API/OnboardingTasks.php
index b3907bb7a23..3e0a679646a 100644
--- a/plugins/woocommerce/src/Admin/API/OnboardingTasks.php
+++ b/plugins/woocommerce/src/Admin/API/OnboardingTasks.php
@@ -11,6 +11,7 @@ use Automattic\WooCommerce\Admin\Features\Features;
use Automattic\WooCommerce\Admin\Features\OnboardingTasks\DeprecatedExtendedTask;
use Automattic\WooCommerce\Admin\Features\OnboardingTasks\TaskLists;
use Automattic\WooCommerce\Enums\ProductStatus;
+use Automattic\WooCommerce\Internal\Admin\Onboarding\MarketplaceTaskExperiment;
use Automattic\WooCommerce\Internal\Admin\Onboarding\OnboardingIndustries;
use Automattic\WooCommerce\Internal\Admin\Onboarding\OnboardingProfile;
use Automattic\WooCommerce\Internal\Utilities\ProductUtil;
@@ -768,9 +769,12 @@ class OnboardingTasks extends \WC_REST_Data_Controller {
$lists = is_array( $task_list_ids ) && count( $task_list_ids ) > 0 ? TaskLists::get_lists_by_ids( $task_list_ids ) : TaskLists::get_lists();
+ $marketplace_task_experiment = wc_get_container()->get( MarketplaceTaskExperiment::class );
+
$json = array_map(
- function ( $list ) { // phpcs:ignore Universal.NamingConventions.NoReservedKeywordParameterNames.listFound
- return $list->sort_tasks()->get_json();
+ function ( $list ) use ( $marketplace_task_experiment ) { // phpcs:ignore Universal.NamingConventions.NoReservedKeywordParameterNames.listFound
+ $marketplace_task_experiment->maybe_move_task_first( $list->sort_tasks() );
+ return $list->get_json();
},
$lists
);
@@ -1000,7 +1004,8 @@ class OnboardingTasks extends \WC_REST_Data_Controller {
}
$update = $task_list->unhide();
- $json = $task_list->get_json();
+ wc_get_container()->get( MarketplaceTaskExperiment::class )->maybe_move_task_first( $task_list );
+ $json = $task_list->get_json();
return rest_ensure_response( $json );
}
diff --git a/plugins/woocommerce/src/Admin/Features/OnboardingTasks/Tasks/ExtendStore.php b/plugins/woocommerce/src/Admin/Features/OnboardingTasks/Tasks/ExtendStore.php
index 69ed8d92a7b..f5cbf7cfdba 100644
--- a/plugins/woocommerce/src/Admin/Features/OnboardingTasks/Tasks/ExtendStore.php
+++ b/plugins/woocommerce/src/Admin/Features/OnboardingTasks/Tasks/ExtendStore.php
@@ -3,6 +3,7 @@
namespace Automattic\WooCommerce\Admin\Features\OnboardingTasks\Tasks;
use Automattic\WooCommerce\Admin\Features\OnboardingTasks\Task;
+use Automattic\WooCommerce\Internal\Admin\Onboarding\MarketplaceTaskExperiment;
/**
* ExtendStore Task
@@ -23,6 +24,16 @@ class ExtendStore extends Task {
* @return string
*/
public function get_title() {
+ $variation = wc_get_container()->get( MarketplaceTaskExperiment::class )->get_variation( $this );
+
+ if ( MarketplaceTaskExperiment::COPY_PAYMENTS_SHIPPING_MARKETING === $variation ) {
+ return __( 'Add payments, shipping and marketing extensions', 'woocommerce' );
+ }
+
+ if ( MarketplaceTaskExperiment::COPY_FREE_AND_PAID === $variation ) {
+ return __( 'Browse free and paid extensions', 'woocommerce' );
+ }
+
return __( 'Enhance your store with extensions', 'woocommerce' );
}
diff --git a/plugins/woocommerce/src/Internal/Admin/Onboarding/MarketplaceTaskExperiment.php b/plugins/woocommerce/src/Internal/Admin/Onboarding/MarketplaceTaskExperiment.php
new file mode 100644
index 00000000000..b14f61251be
--- /dev/null
+++ b/plugins/woocommerce/src/Internal/Admin/Onboarding/MarketplaceTaskExperiment.php
@@ -0,0 +1,172 @@
+<?php
+declare( strict_types = 1 );
+
+namespace Automattic\WooCommerce\Internal\Admin\Onboarding;
+
+use Automattic\WooCommerce\Admin\Features\OnboardingTasks\Task;
+use Automattic\WooCommerce\Admin\Features\OnboardingTasks\TaskList;
+use Automattic\WooCommerce\Admin\Features\OnboardingTasks\TaskLists;
+use Automattic\WooCommerce\Proxies\LegacyProxy;
+use WooCommerce\Admin\Experimental_Abtest;
+
+/**
+ * ExPlat A/B test for the title and position of the Marketplace ("extend-store") task in the "extended" task list.
+ *
+ * @since 11.3.0
+ */
+final class MarketplaceTaskExperiment {
+ /**
+ * ExPlat experiment name.
+ */
+ public const EXPERIMENT_NAME = 'woocommerce_marketplace_task_202611';
+
+ /**
+ * Current title and position.
+ */
+ public const CONTROL = 'control';
+
+ /**
+ * Title "Add payments, shipping and marketing extensions", current position.
+ */
+ public const COPY_PAYMENTS_SHIPPING_MARKETING = 'copy_payments_shipping_marketing';
+
+ /**
+ * Title "Browse free and paid extensions", current position.
+ */
+ public const COPY_FREE_AND_PAID = 'copy_free_and_paid';
+
+ /**
+ * Current title, task moved to the top of the "extended" list.
+ */
+ public const FIRST_POSITION = 'first_position';
+
+ /**
+ * ID of the task under test.
+ */
+ private const TASK_ID = 'extend-store';
+
+ /**
+ * Transient set after a failed ExPlat request, so the next requests skip the call for a while.
+ */
+ private const BACKOFF_TRANSIENT = 'woocommerce_marketplace_task_experiment_backoff';
+
+ /**
+ * 2027-03-01 00:00 UTC. After this, ExPlat is never asked, so stores that don't update stop calling it.
+ */
+ private const END_TIMESTAMP = 1803859200;
+
+ /**
+ * Variation fetched from ExPlat (or its transient) during the current request.
+ *
+ * @var string|null
+ */
+ private ?string $variation = null;
+
+ /**
+ * Get the variation for this request, or control when ExPlat shouldn't be asked.
+ *
+ * @since 11.3.0
+ *
+ * @param Task $task The Marketplace task.
+ * @return string One of the variation constants.
+ */
+ public function get_variation( Task $task ): string {
+ if ( null === $this->variation && $this->should_request_assignment( $task ) ) {
+ $this->variation = $this->request_variation();
+ }
+
+ return $this->variation ?? self::CONTROL;
+ }
+
+ /**
+ * Move the Marketplace task to the top of the "extended" list when the site is in the first_position variation.
+ *
+ * @since 11.3.0
+ *
+ * @param mixed $task_list Task list from TaskLists, which external code can filter, so it may not be a TaskList.
+ * @return void
+ */
+ public function maybe_move_task_first( $task_list ): void {
+ if ( ! $task_list instanceof TaskList || 'extended' !== $task_list->get_list_id() ) {
+ return;
+ }
+
+ $task = $task_list->get_task( self::TASK_ID );
+ if ( ! $task instanceof Task || self::FIRST_POSITION !== $this->get_variation( $task ) ) {
+ return;
+ }
+
+ $other_tasks = array_filter(
+ $task_list->tasks,
+ fn( $other_task ) => $other_task !== $task
+ );
+ $task_list->tasks = array_merge( array( $task ), array_values( $other_tasks ) );
+ }
+
+ /**
+ * Whether to ask ExPlat for an assignment, which logs an exposure.
+ *
+ * @param Task $task The Marketplace task.
+ * @return bool
+ */
+ private function should_request_assignment( Task $task ): bool {
+ if ( $this->has_ended() || ! \WC_Site_Tracking::is_tracking_enabled() || $task->is_dismissed() ) {
+ return false;
+ }
+
+ // Without a tk_ai cookie Experimental_Abtest fails before making a request, so skip it here rather than back off.
+ if ( '' === $this->get_anon_id() || get_transient( self::BACKOFF_TRANSIENT ) ) {
+ return false;
+ }
+
+ if ( $task->is_complete() ) {
+ return false;
+ }
+
+ $task_list = TaskLists::get_list( $task->get_parent_id() );
+ return ! $task_list || $task_list->is_visible();
+ }
+
+ /**
+ * Whether the end date has passed. Reads the time through LegacyProxy so tests can change it.
+ *
+ * @return bool
+ */
+ private function has_ended(): bool {
+ return wc_get_container()->get( LegacyProxy::class )->call_function( 'time' ) >= self::END_TIMESTAMP;
+ }
+
+ /**
+ * Get the Tracks anonymous ID from the tk_ai cookie.
+ *
+ * @return string Empty when the cookie is not set.
+ */
+ private function get_anon_id(): string {
+ return isset( $_COOKIE['tk_ai'] ) ? sanitize_text_field( wp_unslash( $_COOKIE['tk_ai'] ) ) : '';
+ }
+
+ /**
+ * Fetch the assignment from ExPlat (or its transient cache), mapping anything unexpected to control.
+ *
+ * @return string
+ */
+ private function request_variation(): string {
+ $abtest = new Experimental_Abtest( $this->get_anon_id(), 'woocommerce', true );
+
+ try {
+ $variation = $abtest->get_variation( self::EXPERIMENT_NAME );
+ } catch ( \Exception $e ) {
+ // Outside production, get_variation() throws when the request fails instead of returning control.
+ $variation = self::CONTROL;
+ }
+
+ // Experimental_Abtest only caches successful responses, so without a backoff every page load would retry the blocking request.
+ if ( empty( get_transient( 'abtest_variation_' . self::EXPERIMENT_NAME ) ) ) {
+ set_transient( self::BACKOFF_TRANSIENT, 1, HOUR_IN_SECONDS );
+ }
+
+ $known = array( self::COPY_PAYMENTS_SHIPPING_MARKETING, self::COPY_FREE_AND_PAID, self::FIRST_POSITION );
+
+ return in_array( $variation, $known, true ) ? $variation : self::CONTROL;
+ }
+}
diff --git a/plugins/woocommerce/tests/php/src/Internal/Admin/Onboarding/MarketplaceTaskExperimentTest.php b/plugins/woocommerce/tests/php/src/Internal/Admin/Onboarding/MarketplaceTaskExperimentTest.php
new file mode 100644
index 00000000000..a7b884b52a3
--- /dev/null
+++ b/plugins/woocommerce/tests/php/src/Internal/Admin/Onboarding/MarketplaceTaskExperimentTest.php
@@ -0,0 +1,262 @@
+<?php
+declare( strict_types = 1 );
+
+namespace Automattic\WooCommerce\Tests\Internal\Admin\Onboarding;
+
+use Automattic\WooCommerce\Admin\Features\OnboardingTasks\Task;
+use Automattic\WooCommerce\Admin\Features\OnboardingTasks\TaskList;
+use Automattic\WooCommerce\Admin\Features\OnboardingTasks\TaskLists;
+use Automattic\WooCommerce\Internal\Admin\Onboarding\MarketplaceTaskExperiment;
+use Automattic\WooCommerce\Internal\Admin\WCAdminUser;
+use WC_Unit_Test_Case;
+use WP_REST_Request;
+
+/**
+ * Tests for the MarketplaceTaskExperiment class.
+ */
+class MarketplaceTaskExperimentTest extends WC_Unit_Test_Case {
+
+ /**
+ * The System Under Test.
+ *
+ * @var MarketplaceTaskExperiment
+ */
+ private $sut;
+
+ /**
+ * Set up test fixtures.
+ */
+ public function setUp(): void {
+ parent::setUp();
+
+ TaskLists::clear_lists();
+ TaskLists::init_default_lists();
+
+ $this->sut = new MarketplaceTaskExperiment();
+ wc_get_container()->replace( MarketplaceTaskExperiment::class, $this->sut );
+
+ // WP_HTTP_TestCase records every request in $this->http_requests; never reach the live ExPlat API.
+ $this->http_responder = fn() => array(
+ 'response' => array( 'code' => 200 ),
+ 'body' => wp_json_encode(
+ array(
+ 'variations' => array( MarketplaceTaskExperiment::EXPERIMENT_NAME => MarketplaceTaskExperiment::FIRST_POSITION ),
+ 'ttl' => 60,
+ )
+ ),
+ );
+
+ update_option( 'woocommerce_allow_tracking', 'yes' );
+ $_COOKIE['tk_ai'] = 'test-anon-id';
+
+ // Pin the clock before the end date so these tests keep passing after the experiment ends.
+ $this->register_legacy_proxy_function_mocks( array( 'time' => fn() => 1803859200 - 1 ) );
+ }
+
+ /**
+ * Tear down test fixtures.
+ */
+ public function tearDown(): void {
+ try {
+ unset( $_COOKIE['tk_ai'] );
+ $this->reset_legacy_proxy_mocks();
+ wc_get_container()->reset_all_replacements();
+ TaskLists::clear_lists();
+ TaskLists::init_default_lists();
+ } finally {
+ parent::tearDown();
+ }
+ }
+
+ /**
+ * Name of the backoff transient set after a failed request (mirrors the private constant in MarketplaceTaskExperiment).
+ */
+ private const BACKOFF_TRANSIENT = 'woocommerce_marketplace_task_experiment_backoff';
+
+ /**
+ * Get the Marketplace task from the default "extended" list.
+ *
+ * @return Task
+ */
+ private function get_task(): Task {
+ return TaskLists::get_task( 'extend-store', 'extended' );
+ }
+
+ /**
+ * @testdox Should return control without contacting ExPlat when tracking is disabled.
+ */
+ public function test_returns_control_without_request_when_tracking_disabled(): void {
+ update_option( 'woocommerce_allow_tracking', 'no' );
+ set_transient( 'abtest_variation_' . MarketplaceTaskExperiment::EXPERIMENT_NAME, MarketplaceTaskExperiment::COPY_FREE_AND_PAID );
+
+ $this->assertSame( MarketplaceTaskExperiment::CONTROL, $this->sut->get_variation( $this->get_task() ) );
+ $this->assertSame( 'Enhance your store with extensions', $this->get_task()->get_title() );
+ $this->assertCount( 0, $this->http_requests, 'No assignment should be requested without tracking consent' );
+ }
+
+ /**
+ * @testdox Should use the title for the assigned variation.
+ *
+ * @testWith ["copy_payments_shipping_marketing", "Add payments, shipping and marketing extensions"]
+ * ["copy_free_and_paid", "Browse free and paid extensions"]
+ * ["first_position", "Enhance your store with extensions"]
+ * ["unknown_variation", "Enhance your store with extensions"]
+ *
+ * @param string $variation Cached ExPlat assignment.
+ * @param string $expected_title Expected task title.
+ */
+ public function test_title_matches_assigned_variation( string $variation, string $expected_title ): void {
+ set_transient( 'abtest_variation_' . MarketplaceTaskExperiment::EXPERIMENT_NAME, $variation );
+
+ $this->assertSame( $expected_title, $this->get_task()->get_title() );
+ $this->assertCount( 0, $this->http_requests, 'A cached assignment should not trigger a request' );
+ }
+
+ /**
+ * @testdox Should move the task to the top of the extended list only for first_position.
+ *
+ * @testWith ["first_position", true]
+ * ["copy_free_and_paid", false]
+ *
+ * @param string $variation Cached ExPlat assignment.
+ * @param bool $expect_moved Whether the task should be first.
+ */
+ public function test_tasks_endpoint_moves_task_first_for_first_position( string $variation, bool $expect_moved ): void {
+ $default_ids = $this->prepare_extended_list( $variation );
+
+ $request = new WP_REST_Request( 'GET', '/wc-admin/onboarding/tasks' );
+ $request->set_param( 'ids', array( 'extended' ) );
+ $ids = array_column( rest_get_server()->dispatch( $request )->get_data()[0]['tasks'], 'id' );
+
+ $this->assertSame( $expect_moved ? $this->move_task_first( $default_ids ) : $default_ids, $ids );
+ }
+
+ /**
+ * @testdox Should return the task first when the extended list is unhidden for first_position.
+ */
+ public function test_unhide_endpoint_moves_task_first_for_first_position(): void {
+ $default_ids = $this->prepare_extended_list( MarketplaceTaskExperiment::FIRST_POSITION );
+ update_option( TaskList::HIDDEN_OPTION, array( 'extended' ) );
+ $this->assertSame( MarketplaceTaskExperiment::CONTROL, $this->sut->get_variation( $this->get_task() ), 'A hidden list should resolve to control' );
+
+ $data = rest_get_server()->dispatch( new WP_REST_Request( 'POST', '/wc-admin/onboarding/tasks/extended/unhide' ) )->get_data();
+
+ $this->assertSame( $this->move_task_first( $default_ids ), array_column( $data['tasks'], 'id' ) );
+ }
+
+ /**
+ * Seed the assignment, log in as an admin, and return the extended list's default task order.
+ *
+ * @param string $variation Cached ExPlat assignment.
+ * @return string[]
+ */
+ private function prepare_extended_list( string $variation ): array {
+ set_transient( 'abtest_variation_' . MarketplaceTaskExperiment::EXPERIMENT_NAME, $variation );
+ wp_set_current_user( $this->factory->user->create( array( 'role' => 'administrator' ) ) );
+ $default_ids = array_map( fn( $task ) => $task->get_id(), TaskLists::get_list( 'extended' )->get_viewable_tasks() );
+ $this->assertNotSame( 'extend-store', $default_ids[0], 'The task should not be first by default' );
+
+ return $default_ids;
+ }
+
+ /**
+ * Expected order with the Marketplace task first and the rest unchanged.
+ *
+ * @param string[] $ids Default task order.
+ * @return string[]
+ */
+ private function move_task_first( array $ids ): array {
+ return array_merge( array( 'extend-store' ), array_values( array_diff( $ids, array( 'extend-store' ) ) ) );
+ }
+
+ /**
+ * @testdox Should fetch the assignment from ExPlat and cache it when no transient is set.
+ *
+ * @testWith ["copy_free_and_paid", "copy_free_and_paid"]
+ * [null, "control"]
+ *
+ * @param string|null $assigned Variation in the ExPlat response, or null for an empty response.
+ * @param string $expected Expected variation.
+ */
+ public function test_fetches_assignment_from_explat( ?string $assigned, string $expected ): void {
+ $body = null === $assigned
+ ? array(
+ 'variations' => new \stdClass(),
+ 'assignments' => new \stdClass(),
+ 'ttl' => 7200,
+ )
+ : array(
+ 'variations' => array( MarketplaceTaskExperiment::EXPERIMENT_NAME => $assigned ),
+ 'ttl' => 3600,
+ );
+ $this->http_responder = fn() => array(
+ 'response' => array( 'code' => 200 ),
+ 'body' => wp_json_encode( $body ),
+ );
+
+ $this->assertSame( $expected, $this->sut->get_variation( $this->get_task() ) );
+ $this->assertCount( 1, $this->http_requests );
+ wp_parse_str( (string) wp_parse_url( $this->http_requests[0]['url'], PHP_URL_QUERY ), $args );
+ $this->assertSame( MarketplaceTaskExperiment::EXPERIMENT_NAME, $args['experiment_name'] );
+ $this->assertSame( 'test-anon-id', $args['anon_id'] );
+ $this->assertSame( $expected, get_transient( 'abtest_variation_' . MarketplaceTaskExperiment::EXPERIMENT_NAME ) );
+ $this->assertFalse( get_transient( self::BACKOFF_TRANSIENT ) );
+ }
+
+ /**
+ * @testdox Should fall back to control and stop requesting for a while when the request fails.
+ *
+ * @testWith ["wp_error"]
+ * ["server_error"]
+ *
+ * @param string $failure Kind of failed response.
+ */
+ public function test_backs_off_when_request_fails( string $failure ): void {
+ $this->http_responder = 'wp_error' === $failure
+ ? fn() => new \WP_Error( 'http_request_failed', 'Offline' )
+ : fn() => array(
+ 'response' => array( 'code' => 500 ),
+ 'body' => 'Internal Server Error',
+ );
+
+ $this->assertSame( MarketplaceTaskExperiment::CONTROL, $this->sut->get_variation( $this->get_task() ) );
+ $this->assertNotEmpty( get_transient( self::BACKOFF_TRANSIENT ), 'A failed request should set the backoff' );
+
+ $this->assertSame( MarketplaceTaskExperiment::CONTROL, ( new MarketplaceTaskExperiment() )->get_variation( $this->get_task() ) );
+ $this->assertCount( 1, $this->http_requests, 'No request should be made during the backoff' );
+ }
+
+ /**
+ * @testdox Should not request an assignment when the task is dismissed, complete, its list is hidden, there is no tk_ai cookie, tracking is filtered off, or the experiment has ended.
+ *
+ * @testWith ["dismissed"]
+ * ["visited"]
+ * ["hidden_list"]
+ * ["no_anon_id"]
+ * ["tracking_filtered_off"]
+ * ["ended"]
+ *
+ * @param string $state Task state that should skip the request.
+ */
+ public function test_skips_request_for_inactive_task( string $state ): void {
+ if ( 'dismissed' === $state ) {
+ update_option( Task::DISMISSED_OPTION, array( 'extend-store' ) );
+ } elseif ( 'visited' === $state ) {
+ $user_id = $this->factory->user->create( array( 'role' => 'administrator' ) );
+ wp_set_current_user( $user_id );
+ WCAdminUser::update_user_data_field( $user_id, 'task_list_tracked_started_tasks', wp_json_encode( array( 'extend-store' => 1 ) ) );
+ } elseif ( 'hidden_list' === $state ) {
+ update_option( TaskList::HIDDEN_OPTION, array( 'extended' ) );
+ } elseif ( 'no_anon_id' === $state ) {
+ unset( $_COOKIE['tk_ai'] );
+ } elseif ( 'tracking_filtered_off' === $state ) {
+ add_filter( 'woocommerce_apply_user_tracking', '__return_false' );
+ } else {
+ $this->register_legacy_proxy_function_mocks( array( 'time' => fn() => 1803859200 ) );
+ }
+
+ $this->assertSame( MarketplaceTaskExperiment::CONTROL, $this->sut->get_variation( $this->get_task() ) );
+ $this->assertCount( 0, $this->http_requests, 'No assignment should be requested' );
+ $this->assertFalse( get_transient( self::BACKOFF_TRANSIENT ), 'A skipped request should not set the backoff' );
+ }
+}