From 34bbd04c0a2de68a6f0100165b0cabc08cf91d82 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sebastian=20H=C3=A4hnlein?=
 <50739746+Sebastian-Haehnlein@users.noreply.github.com>
Date: Thu, 9 Jan 2020 14:16:25 +0100
Subject: [PATCH] feat: serve robots.txt from universal server (#66)

---
 .gitignore        |  3 ---
 dist/.gitignore   |  5 +++++
 package-lock.json |  5 +++++
 package.json      |  1 +
 server.ts         | 27 +++++++++++++++++++++++++++
 5 files changed, 38 insertions(+), 3 deletions(-)
 create mode 100644 dist/.gitignore

diff --git a/.gitignore b/.gitignore
index 2d9829e2c7..274ef2ed46 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,9 +1,6 @@
 # See http://help.github.com/ignore-files/ for more about ignoring files.
 
 # compiled or cached output
-/dist/**/*
-!/dist/*.sh
-!/dist/healthcheck.js
 /tmp
 /out-tsc
 /bin
diff --git a/dist/.gitignore b/dist/.gitignore
new file mode 100644
index 0000000000..7fd7aa6c38
--- /dev/null
+++ b/dist/.gitignore
@@ -0,0 +1,5 @@
+/**/*
+!/.gitignore
+!/*.sh
+!/healthcheck.js
+!/robots.txt
diff --git a/package-lock.json b/package-lock.json
index 1235289470..e304fb638b 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -6367,6 +6367,11 @@
         }
       }
     },
+    "express-robots-txt": {
+      "version": "0.4.1",
+      "resolved": "https://registry.npmjs.org/express-robots-txt/-/express-robots-txt-0.4.1.tgz",
+      "integrity": "sha512-qVffRP/YmwZEISWB/gAGfJ+Y85sUKcjGXtryCOZFRyKOvF2b/JaG2xpPKVT9NeNOMZYvNq000qHp/oBtLYgTnQ=="
+    },
     "extend": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz",
diff --git a/package.json b/package.json
index 7f283b1ccc..72833621c7 100644
--- a/package.json
+++ b/package.json
@@ -73,6 +73,7 @@
     "conventional-changelog-cli": "^2.0.21",
     "core-js": "^2.6.9",
     "express": "^4.17.1",
+    "express-robots-txt": "0.4.1",
     "lodash-es": "^4.17.15",
     "ng-recaptcha": "^5.0.0",
     "ng2-validation": "^4.2.0",
diff --git a/server.ts b/server.ts
index fb101f5cb0..f27fe34f7f 100644
--- a/server.ts
+++ b/server.ts
@@ -20,6 +20,8 @@ import 'zone.js/dist/zone-node';
 
 import * as express from 'express';
 import { join } from 'path';
+import * as robots from 'express-robots-txt';
+import * as fs from 'fs';
 
 const logging = !!process.env.LOGGING;
 
@@ -44,6 +46,31 @@ app.engine(
 app.set('view engine', 'html');
 app.set('views', join(DIST_FOLDER, 'browser'));
 
+// seo robots.txt
+const pathToRobotsTxt = join(DIST_FOLDER, 'robots.txt');
+if (fs.existsSync(pathToRobotsTxt)) {
+  app.use(robots(pathToRobotsTxt));
+} else {
+  app.use(
+    robots({
+      UserAgent: '*',
+      Disallow: [
+        '/error',
+        '/account',
+        '/compare',
+        '/recently',
+        '/basket',
+        '/checkout',
+        '/register',
+        '/login',
+        '/logout',
+        '/forgotPassword',
+        '/contact',
+      ],
+    })
+  );
+}
+
 // Serve static files from /browser
 app.get(
   '*.*',