From d4b049dc34896b0be3479a8cf1171ea2ae6ec809 Mon Sep 17 00:00:00 2001 From: aescling Date: Sat, 31 Dec 2022 22:00:30 -0500 Subject: [PATCH] Actually prevent search engine indexing Disabling visibility via robots.txt prevents purroper indexers, like googlebot, from detecting X-Robots-Tag or the like is on a page. This also adds an X-Robots-Tag header to all requests. --- config/environments/production.rb | 3 ++- public/robots.txt | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/config/environments/production.rb b/config/environments/production.rb index dd73bb4e1..c80e92681 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -124,7 +124,8 @@ Rails.application.configure do 'X-XSS-Protection' => '0', 'Permissions-Policy' => 'interest-cohort=()', 'Referrer-Policy' => 'same-origin', - 'X-Clacks-Overhead' => 'GNU Natalie Nguyen' + 'X-Clacks-Overhead' => 'GNU Natalie Nguyen', + 'X-Robots-Tag' => 'noindex, nofollow, noarchive' } config.x.otp_secret = ENV.fetch('OTP_SECRET') diff --git a/public/robots.txt b/public/robots.txt index 78a0ccab9..771bf2160 100644 --- a/public/robots.txt +++ b/public/robots.txt @@ -1,4 +1,5 @@ # See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file User-agent: * -Disallow: / +Disallow: /media_proxy/ +Disallow: /interact/ -- 2.47.3