Actually prevent search engine indexing

Disabling visibility via robots.txt prevents purroper indexers, like
googlebot, from detecting X-Robots-Tag or the like is on a page. This
also adds an X-Robots-Tag header to all requests.

diff --git a/config/environments/production.rb b/config/environments/production.rb
index dd73bb4e1b22fd870b0fb2d4a93c9aa6c4459b87..c80e92681588ba4f9d4e429087d8d5e3f6830ccb 100644 (file)
--- a/config/environments/production.rb
+++ b/config/environments/production.rb
@@ -124,7 +124,8 @@ Rails.application.configure do
     'X-XSS-Protection'        => '0',
     'Permissions-Policy'      => 'interest-cohort=()',
     'Referrer-Policy'         => 'same-origin',
-    'X-Clacks-Overhead' => 'GNU Natalie Nguyen'
+    'X-Clacks-Overhead' => 'GNU Natalie Nguyen',
+    'X-Robots-Tag' => 'noindex, nofollow, noarchive'
   }
 
   config.x.otp_secret = ENV.fetch('OTP_SECRET')

diff --git a/public/robots.txt b/public/robots.txt
index 78a0ccab9c9622fe2d4a48d2e40c7580ed722891..771bf2160b4e3bdc65981d3680ef1ae78925f79d 100644 (file)
--- a/public/robots.txt
+++ b/public/robots.txt
@@ -1,4 +1,5 @@
 # See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
 
 User-agent: *
-Disallow: /
+Disallow: /media_proxy/
+Disallow: /interact/