Skip to content

Commit d36aad3

Browse files
haqadnhaqadn
authored
SEO: Keep WordPress.com's robots.txt rules in the User-agent: * group (#54010)
* SEO: Add AI crawler groups after WordPress.com's robots.txt lines WordPress.com Simple appends bare Disallow lines (wp-login.php, public.api, ...) to robots.txt at priority 11, with no User-agent line of their own. With the AI crawler groups added at priority 10, those lines landed in the last group (User-agent: Amazonbot), so search engines no longer got them. The filter now runs at priority 12. * SEO: Drop the stale filter-order note from the opt-out docblock * SEO: Correct the robots.txt docblocks after review
1 parent 9f10088 commit d36aad3

5 files changed

Lines changed: 51 additions & 12 deletions

File tree

‎projects/packages/seo/changelog/fix-seo-ai-crawlers-robots-priority‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Significance: patch
2+
Type: changed
3+
Comment: GEO: Add the AI crawler groups to robots.txt after WordPress.com Simple's own Disallow lines, so those lines stay in the `User-agent: *` group. The new SEO page is still gated.
4+

‎projects/packages/seo/src/class-ai-crawlers.php‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -37,13 +37,21 @@ class Ai_Crawlers {
3737
*/
3838
const OPTION = 'jetpack_seo_ai_crawler_overrides';
3939

40+
/**
41+
* The `robots_txt` priority: after WordPress.com Simple appends bare `Disallow`
42+
* lines at 11, which would otherwise land in the last AI crawler group.
43+
*
44+
* @var int
45+
*/
46+
const ROBOTS_TXT_PRIORITY = 12;
47+
4048
/**
4149
* Wire the robots.txt filter.
4250
*
4351
* @return void
4452
*/
4553
public static function init() {
46-
add_filter( 'robots_txt', array( __CLASS__, 'append_directives' ), 10, 2 );
54+
add_filter( 'robots_txt', array( __CLASS__, 'append_directives' ), self::ROBOTS_TXT_PRIORITY, 2 );
4755
}
4856

4957
/**
@@ -154,7 +162,7 @@ private static function is_blocked_by_default( $slug ) {
154162
/**
155163
* Whether WordPress.com's site-wide data-sharing opt-out is enabled.
156164
*
157-
* Its later `robots_txt` filter remains authoritative over per-bot settings.
165+
* Its `robots_txt` filter remains authoritative over per-bot settings.
158166
*
159167
* @return bool
160168
*/
@@ -321,17 +329,15 @@ public static function is_path_based_multisite() {
321329
* Append `Disallow: /` blocks for each blocked AI crawler to robots.txt.
322330
*
323331
* @param string $output The robots.txt content assembled so far.
324-
* @param bool $public Whether the site is public (`blog_public`). When
325-
* false WordPress already disallows everything, but the
326-
* explicit per-bot blocks are still valid and harmless.
332+
* @param bool $public Whether the site is public (`blog_public`). Unused: the
333+
* per-bot blocks apply either way.
327334
* @return string The robots.txt content with AI-crawler directives appended.
328335
*/
329336
public static function append_directives( $output, $public ) {
330337
unset( $public );
331338

332-
// WordPress.com's existing privacy filter owns the site-wide opt-out and
333-
// runs later at priority 12. Defer to it instead of emitting duplicate
334-
// directives from this per-bot filter.
339+
// WordPress.com's existing privacy filter owns the site-wide opt-out. Defer
340+
// to it instead of emitting duplicate directives from this per-bot filter.
335341
if ( self::is_path_based_multisite() || self::has_data_sharing_opt_out() ) {
336342
return $output;
337343
}

‎projects/packages/seo/tests/php/AiCrawlersTest.php‎

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ protected function setUp(): void {
3333
$this->blog_public = get_option( 'blog_public', null );
3434
delete_option( Ai_Crawlers::OPTION );
3535
delete_option( 'wpcom_data_sharing_opt_out' );
36-
remove_filter( 'robots_txt', array( Ai_Crawlers::class, 'append_directives' ), 10 );
36+
remove_filter( 'robots_txt', array( Ai_Crawlers::class, 'append_directives' ), Ai_Crawlers::ROBOTS_TXT_PRIORITY );
3737
remove_filter(
3838
'robots_txt',
3939
'Automattic\\Jetpack\\Jetpack_Mu_Wpcom\\Blog_Privacy\\robots_txt',
@@ -49,7 +49,7 @@ protected function setUp(): void {
4949
protected function tearDown(): void {
5050
delete_option( Ai_Crawlers::OPTION );
5151
delete_option( 'wpcom_data_sharing_opt_out' );
52-
remove_filter( 'robots_txt', array( Ai_Crawlers::class, 'append_directives' ), 10 );
52+
remove_filter( 'robots_txt', array( Ai_Crawlers::class, 'append_directives' ), Ai_Crawlers::ROBOTS_TXT_PRIORITY );
5353
remove_filter(
5454
'robots_txt',
5555
'Automattic\\Jetpack\\Jetpack_Mu_Wpcom\\Blog_Privacy\\robots_txt',
@@ -243,6 +243,31 @@ public function test_append_directives_unchanged_when_nothing_blocked() {
243243
$this->assertSame( $input, Ai_Crawlers::append_directives( $input, true ) );
244244
}
245245

246+
/**
247+
* Bare `Disallow` lines another filter appends first stay in the `User-agent: *` group
248+
* rather than the last AI crawler group.
249+
*
250+
* @return void
251+
*/
252+
public function test_bare_lines_appended_before_ours_stay_in_the_wildcard_group() {
253+
$append_bare_line = static function ( $output ) {
254+
return $output . "Disallow: /wp-login.php\n";
255+
};
256+
Ai_Crawlers::init();
257+
add_filter( 'robots_txt', $append_bare_line, 11 );
258+
update_option( 'blog_public', 1 );
259+
260+
try {
261+
$output = apply_filters( 'robots_txt', "User-agent: *\nDisallow: /wp-admin/\n", true );
262+
} finally {
263+
remove_filter( 'robots_txt', $append_bare_line, 11 );
264+
}
265+
266+
$before_ai_groups = strstr( $output, '# AI crawlers blocked via Jetpack SEO.', true );
267+
$this->assertIsString( $before_ai_groups );
268+
$this->assertStringContainsString( 'Disallow: /wp-login.php', $before_ai_groups );
269+
}
270+
246271
/**
247272
* The SEO and WordPress.com privacy filters agree when both run.
248273
*
@@ -271,7 +296,7 @@ public function test_combined_robots_filters_respect_data_sharing_opt_out() {
271296

272297
$output = apply_filters( 'robots_txt', 'TEST', true );
273298

274-
$this->assertSame( 10, has_filter( 'robots_txt', array( Ai_Crawlers::class, 'append_directives' ) ) );
299+
$this->assertSame( Ai_Crawlers::ROBOTS_TXT_PRIORITY, has_filter( 'robots_txt', array( Ai_Crawlers::class, 'append_directives' ) ) );
275300
$this->assertSame(
276301
12,
277302
has_filter(

‎projects/packages/seo/tests/php/InitializerTest.php‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -228,7 +228,7 @@ private function init_and_check_paid_services() {
228228
// The services self-hook on init(), so a stale registration from an earlier
229229
// test would mask a missing one here.
230230
remove_action( 'template_redirect', array( Llms_Txt::class, 'maybe_serve' ) );
231-
remove_filter( 'robots_txt', array( Ai_Crawlers::class, 'append_directives' ), 10 );
231+
remove_filter( 'robots_txt', array( Ai_Crawlers::class, 'append_directives' ), Ai_Crawlers::ROBOTS_TXT_PRIORITY );
232232
remove_action( 'after_setup_theme', array( Canonical_Urls::class, 'load_module' ) );
233233

234234
try {

‎projects/plugins/jetpack/changelog/fix-seo-ai-crawlers-robots-priority‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Significance: patch
2+
Type: other
3+
Comment: SEO: Add the AI crawler groups to robots.txt after WordPress.com Simple's own Disallow lines, so those lines stay in the `User-agent: *` group. The new SEO page is still gated.
4+

0 commit comments

Comments
 (0)