From 468761253b5f6847f58788209e8d24c0bd1301a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Javi=20Mart=C3=ADn?= <javim@elretirao.net>
Date: Fri, 30 Sep 2022 16:04:56 +0200
Subject: [PATCH] Add per-tenant sitemap to robots.txt file

While we ping some search engines (currently, only Google) when
generating the sitemap files, we weren't telling search engines
accessing through the `robots.txt` file where to find the sitemap. Now
we're doing so, using the right sitemap file for the right tenant.
---
 app/controllers/robots_controller.rb          |  7 +++++++
 .../views/robots/index.text.erb               |  6 ++++++
 config/routes.rb                              |  1 +
 spec/system/robots_spec.rb                    | 19 +++++++++++++++++++
 4 files changed, 33 insertions(+)
 create mode 100644 app/controllers/robots_controller.rb
 rename public/robots.txt => app/views/robots/index.text.erb (69%)
 create mode 100644 spec/system/robots_spec.rb

diff --git a/app/controllers/robots_controller.rb b/app/controllers/robots_controller.rb
new file mode 100644
index 000000000..2d445fa01
--- /dev/null
+++ b/app/controllers/robots_controller.rb
@@ -0,0 +1,7 @@
+class RobotsController < ApplicationController
+  skip_authorization_check
+
+  def index
+    respond_to :text
+  end
+end
diff --git a/public/robots.txt b/app/views/robots/index.text.erb
similarity index 69%
rename from public/robots.txt
rename to app/views/robots/index.text.erb
index 9327c9c1b..8a432d78d 100644
--- a/public/robots.txt
+++ b/app/views/robots/index.text.erb
@@ -13,3 +13,9 @@ Disallow: /*?*search
 Disallow: /*?*locale-switcher
 Disallow: /*?*filter
 Disallow: user_id
+
+<% if Tenant.default? %>
+Sitemap: <%= "#{root_url}sitemap.xml" %>
+<% else %>
+Sitemap: <%= "#{root_url}tenants/#{Tenant.current_schema}/sitemap.xml" %>
+<% end %>
diff --git a/config/routes.rb b/config/routes.rb
index d216d1c81..29a4083c6 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -30,6 +30,7 @@ Rails.application.routes.draw do
   root "welcome#index"
   get "/welcome", to: "welcome#welcome"
   get "/consul.json", to: "installation#details"
+  get "robots.txt", to: "robots#index"
 
   resources :stats, only: [:index]
   resources :images, only: [:destroy]
diff --git a/spec/system/robots_spec.rb b/spec/system/robots_spec.rb
new file mode 100644
index 000000000..3e728f24b
--- /dev/null
+++ b/spec/system/robots_spec.rb
@@ -0,0 +1,19 @@
+require "rails_helper"
+
+describe "robots.txt" do
+  scenario "uses the default sitemap for the default tenant" do
+    visit "/robots.txt"
+
+    expect(page).to have_content "Sitemap: #{app_host}/sitemap.xml"
+  end
+
+  scenario "uses a different sitemap for other tenants" do
+    create(:tenant, schema: "cyborgs")
+
+    with_subdomain("cyborgs") do
+      visit "/robots.txt"
+
+      expect(page).to have_content "Sitemap: http://cyborgs.lvh.me:#{app_port}/tenants/cyborgs/sitemap.xml"
+    end
+  end
+end