feat: serve robots.txt from universal server (#66)

.gitignore

@@ -1,9 +1,6 @@
 # See http://help.github.com/ignore-files/ for more about ignoring files.
 
 # compiled or cached output
-/dist/**/*
-!/dist/*.sh
-!/dist/healthcheck.js
 /tmp
 /out-tsc
 /bin

dist/.gitignore

@@ -0,0 +1,5 @@
+/**/*
+!/.gitignore
+!/*.sh
+!/healthcheck.js
+!/robots.txt

package.json

@@ -73,6 +73,7 @@
     "conventional-changelog-cli": "^2.0.21",
     "core-js": "^2.6.9",
     "express": "^4.17.1",
+    "express-robots-txt": "0.4.1",
     "lodash-es": "^4.17.15",
     "ng-recaptcha": "^5.0.0",
     "ng2-validation": "^4.2.0",

server.ts

@@ -20,6 +20,8 @@ import 'zone.js/dist/zone-node';
 
 import * as express from 'express';
 import { join } from 'path';
+import * as robots from 'express-robots-txt';
+import * as fs from 'fs';
 
 const logging = !!process.env.LOGGING;
 
@@ -44,6 +46,31 @@ app.engine(
 app.set('view engine', 'html');
 app.set('views', join(DIST_FOLDER, 'browser'));
 
+// seo robots.txt
+const pathToRobotsTxt = join(DIST_FOLDER, 'robots.txt');
+if (fs.existsSync(pathToRobotsTxt)) {
+  app.use(robots(pathToRobotsTxt));
+} else {
+  app.use(
+    robots({
+      UserAgent: '*',
+      Disallow: [
+        '/error',
+        '/account',
+        '/compare',
+        '/recently',
+        '/basket',
+        '/checkout',
+        '/register',
+        '/login',
+        '/logout',
+        '/forgotPassword',
+        '/contact',
+      ],
+    })
+  );
+}
+
 // Serve static files from /browser
 app.get(
   '*.*',