From ac235c37e2d7112efbc182e0179cb27e75ef09a1 Mon Sep 17 00:00:00 2001
From: Rigel Kent <sendmemail@rigelk.eu>
Date: Tue, 15 May 2018 00:29:40 +0200
Subject: [PATCH] robots.txt config and route

---
 config/default.yaml              | 4 ++++
 config/production.yaml.example   | 4 ++++
 server/controllers/static.ts     | 6 ++++++
 server/initializers/checker.ts   | 2 +-
 server/initializers/constants.ts | 3 ++-
 5 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/config/default.yaml b/config/default.yaml
index 2826e76f8..995638281 100644
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -90,6 +90,10 @@ instance:
   customizations:
     javascript: '' # Directly your JavaScript code (without <script> tags). Will be eval at runtime
     css: '' # Directly your CSS code (without <style> tags). Will be injected at runtime
+  # Robot.txt rules. To allow robots to crawl your instance and allow indexation of your site, remove the '/'
+  robots: |
+    User-agent: *
+    Disallow: /
 
 services:
   # Cards configuration to format video in Twitter
diff --git a/config/production.yaml.example b/config/production.yaml.example
index a6f1740fe..71344fa35 100644
--- a/config/production.yaml.example
+++ b/config/production.yaml.example
@@ -106,6 +106,10 @@ instance:
   customizations:
     javascript: '' # Directly your JavaScript code (without <script> tags). Will be eval at runtime
     css: '' # Directly your CSS code (without <style> tags). Will be injected at runtime
+  # Robot.txt rules. To allow robots to crawl your instance and allow indexation of your site, remove the '/'
+  robots: |
+    User-agent: *
+    Disallow: /
 
 services:
   # Cards configuration to format video in Twitter
diff --git a/server/controllers/static.ts b/server/controllers/static.ts
index eece9c06b..c1bf384a4 100644
--- a/server/controllers/static.ts
+++ b/server/controllers/static.ts
@@ -44,6 +44,12 @@ staticRouter.use(
   asyncMiddleware(getPreview)
 )
 
+// robots.txt service
+staticRouter.get('/robots.txt', (req: express.Request, res: express.Response) => {
+  res.type('text/plain')
+  return res.send(CONFIG.INSTANCE.ROBOTS)
+})
+
 // ---------------------------------------------------------------------------
 
 export {
diff --git a/server/initializers/checker.ts b/server/initializers/checker.ts
index 9bf53e940..c11dc7a89 100644
--- a/server/initializers/checker.ts
+++ b/server/initializers/checker.ts
@@ -29,7 +29,7 @@ function checkMissedConfig () {
     'user.video_quota',
     'cache.previews.size', 'admin.email', 'signup.enabled', 'signup.limit', 'transcoding.enabled', 'transcoding.threads',
     'instance.name', 'instance.short_description', 'instance.description', 'instance.terms', 'instance.default_client_route',
-    'instance.default_nsfw_policy',
+    'instance.default_nsfw_policy', 'instance.robots',
     'services.twitter.username', 'services.twitter.whitelisted'
   ]
   const miss: string[] = []
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 5a7d91160..51d272895 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -180,7 +180,8 @@ const CONFIG = {
     CUSTOMIZATIONS: {
       get JAVASCRIPT () { return config.get<string>('instance.customizations.javascript') },
       get CSS () { return config.get<string>('instance.customizations.css') }
-    }
+    },
+    get ROBOTS () { return config.get<string>('instance.robots') }
   },
   SERVICES: {
     TWITTER: {
-- 
2.25.1