From 2752eed1458becdc8fe0e9e38480bd342eed7836 Mon Sep 17 00:00:00 2001
From: Andrew Marcuse <fileformat@gmail.com>
Date: Wed, 25 Sep 2024 15:54:15 -0400
Subject: [PATCH] Dynamic robots.txt so only www gets indexed

---
 public/robots.txt           | 13 -------------
 src/app/robots.txt/route.ts | 26 ++++++++++++++++++++++++++
 2 files changed, 26 insertions(+), 13 deletions(-)
 delete mode 100644 public/robots.txt
 create mode 100644 src/app/robots.txt/route.ts

diff --git a/public/robots.txt b/public/robots.txt
deleted file mode 100644
index 6bc7fd5..0000000
--- a/public/robots.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-#
-# almost empty: everything at RegexPlanet is indexable!
-#
-Sitemap: https://www.regexplanet.com/sitemap.xml
-
-User-agent: *
-Disallow: /honeypot.txt
-
-#
-# bandwidth hog: http://www.majestic12.co.uk/projects/dsearch/mj12bot.php
-#
-User-agent: MJ12bot
-Crawl-Delay: 120
diff --git a/src/app/robots.txt/route.ts b/src/app/robots.txt/route.ts
new file mode 100644
index 0000000..e78e244
--- /dev/null
+++ b/src/app/robots.txt/route.ts
@@ -0,0 +1,26 @@
+export async function GET(request: Request) {
+
+    let robotsTxt = `#
+# robots.txt - not running on www yet, so don't index anything
+#
+User-agent: *
+Disallow: /
+`;
+
+    if (request.headers.get("Host") === "www.regexplanet.com") {
+        robotsTxt = `#
+# almost empty: everything at RegexPlanet is indexable!
+#
+Sitemap: https://www.regexplanet.com/sitemap.xml
+
+User-agent: *
+Disallow: /honeypot.txt
+`
+    }
+
+    return new Response(robotsTxt, {
+      headers: {
+        "Content-Type": "text/plain; charset=utf-8",
+      },
+    });
+}