Robots: Add noindex to 404 pages

Author: Sukhendu2002

src/wp-includes/default-filters.php

@@ -277,6 +277,7 @@
 add_filter( 'wp_robots', 'wp_robots_noindex' );
 add_filter( 'wp_robots', 'wp_robots_noindex_embeds' );
 add_filter( 'wp_robots', 'wp_robots_noindex_search' );
+add_filter( 'wp_robots', 'wp_robots_noindex_404' );
 add_filter( 'wp_robots', 'wp_robots_max_image_preview_large' );
 
 // Mark site as no longer fresh.

src/wp-includes/functions.php

@@ -3930,6 +3930,7 @@ function _default_wp_die_handler( $message, $title = '', $args = array() ) {
 			// Prevent warnings because of $wp_query not existing.
 			remove_filter( 'wp_robots', 'wp_robots_noindex_embeds' );
 			remove_filter( 'wp_robots', 'wp_robots_noindex_search' );
+			remove_filter( 'wp_robots', 'wp_robots_noindex_404' );
 			wp_robots();
 		}
 		?>

src/wp-includes/robots-template.php

@@ -123,6 +123,32 @@ function wp_robots_noindex_search( array $robots ) {
 	return $robots;
 }
 
+/**
+ * Adds `noindex` to the robots meta tag if a 404 error has occurred.
+ *
+ * If a 404 error has occurred then noindex will be output to
+ * tell web robots not to index the page content. Add this to the
+ * {@see 'wp_robots'} filter.
+ *
+ * Typical usage is as a {@see 'wp_robots'} callback:
+ *
+ *     add_filter( 'wp_robots', 'wp_robots_noindex_404' );
+ *
+ * @since x.x.x
+ *
+ * @see wp_robots_no_robots()
+ *
+ * @param array $robots Associative array of robots directives.
+ * @return array Filtered robots directives.
+ */
+function wp_robots_noindex_404( array $robots ) {
+	if ( is_404() ) {
+		return wp_robots_no_robots( $robots );
+	}
+
+	return $robots;
+}
+
 /**
  * Adds `noindex` to the robots meta tag.
  *

tests/phpunit/tests/robots.php

@@ -11,6 +11,7 @@ class Tests_Robots extends WP_UnitTestCase {
 	public function set_up() {
 		parent::set_up();
 
+		$this->set_permalink_structure( '/%postname%/' );
 		remove_all_filters( 'wp_robots' );
 	}
 
@@ -144,6 +145,30 @@ public function test_wp_robots_non_search_page() {
 		$this->assertStringNotContainsString( 'noindex', $output );
 	}
 
+	/**
+	 * @ticket 58751
+	 * @covers ::wp_robots_noindex_404
+	 */
+	public function test_wp_robots_404_page() {
+		add_filter( 'wp_robots', 'wp_robots_noindex_404' );
+		$this->go_to( '/this-page-does-not-exist' );
+
+		$output = get_echo( 'wp_robots' );
+		$this->assertStringContainsString( 'noindex', $output );
+	}
+
+	/**
+	 * @ticket 58751
+	 * @covers ::wp_robots_noindex_404
+	 */
+	public function test_wp_robots_non_404_page() {
+		add_filter( 'wp_robots', 'wp_robots_noindex_404' );
+		$this->go_to( home_url() );
+
+		$output = get_echo( 'wp_robots' );
+		$this->assertStringNotContainsString( 'noindex', $output );
+	}
+
 	public function add_noindex_directive( array $robots ) {
 		$robots['noindex'] = true;
 		return $robots;