restrict AI agents from crawling the website

nisrulz · Mar 15, 2024 · 9b0cc7e · 9b0cc7e
1 parent e31aee8
commit 9b0cc7e
Showing 1 changed file with 29 additions and 3 deletions.
diff --git a/public/robots.txt b/public/robots.txt
@@ -1,5 +1,31 @@
 # www.robotstxt.org/
 
-# Allow crawling of all content
-User-agent: *
-Disallow:
+# List generated from https://darkvisitors.com/
+
+User-agent: cohere-ai
+Disallow: /
+
+User-agent: anthropic-ai
+Disallow: /
+
+User-agent: Bytespider
+Disallow: /
+
+User-agent: CCBot
+Disallow: /
+
+User-agent: FacebookBot
+Disallow: /
+
+User-agent: Google-Extended
+User-agent: AdsBot-Google
+Disallow: /
+
+User-agent: GPTBot
+Disallow: /
+
+User-agent: ChatGPT-User
+Disallow: /
+
+User-agent: omgili
+Disallow: /