feat(#9): add robot.txt and .htaccess generation

moiety-studio · Dec 27, 2024 · 62c8fc3 · 62c8fc3
1 parent da19c39
commit 62c8fc3
Show file tree

Hide file tree

Showing 6 changed files with 27 additions and 46 deletions.
diff --git a/.eleventy.js b/.eleventy.js
@@ -48,7 +48,6 @@ export default function(eleventyConfig) {
     eleventyConfig.addPassthroughCopy("_source/assets/fonts/")
     eleventyConfig.addPassthroughCopy("_source/assets/app.js")
     eleventyConfig.addPassthroughCopy("_source/assets/core.css")
-    eleventyConfig.addPassthroughCopy({"_source/_meta/robots.txt": "/"})
 
     // 11ty Settings --------------------
     return {

diff --git a/_source/_data/bots.json b/_source/_data/bots.json
@@ -0,0 +1 @@
+["AI2Bot","Ai2Bot-Dolma","Amazonbot","anthropic-ai","Applebot","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","Claude-Web","ClaudeBot","cohere-ai","Diffbot","DuckAssistBot","FacebookBot","FriendlyCrawler","Google-Extended","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","Meta-ExternalAgent","Meta-ExternalFetcher","OAI-SearchBot","omgili","omgilibot","PanguBot","PerplexityBot","PetalBot","Scrapy","Sidetrade indexer bot","Timpibot","VelenPublicWebCrawler","Webzio-Extended","YouBot"]
diff --git a/_source/_meta/htaccess.njk b/_source/_meta/htaccess.njk
@@ -0,0 +1,12 @@
+---
+permalink: .htaccess
+---
+
+<IfModule mod_rewrite.c>
+RewriteEngine on
+RewriteBase /
+
+# block “AI” bots
+RewriteCond %{HTTP_USER_AGENT} ({{ bots | join("|") }}) [NC]
+RewriteRule ^ – [F]
+</IfModule>
diff --git a/_source/_meta/robots.njk b/_source/_meta/robots.njk
@@ -0,0 +1,10 @@
+---
+permalink: robots.txt
+---
+
+User-agent: *
+Disallow:
+
+{% for bot in bots %}User-agent: {{ bot }}
+{% endfor %}
+Disallow: /
diff --git a/_source/_meta/robots.txt b/_source/_meta/robots.txt
diff --git a/_source/root/about.md b/_source/root/about.md
@@ -16,6 +16,10 @@ Elf was built with accessibility in mind. It uses semantic HTML, has contrasting
 
 - CSS is minimised (with the help of [postcss-minify](https://www.npmjs.com/package/postcss-minify))
 
+## Security
+
+- AI scraping is hindered via `robots.txt` and  a `.htaccess` file.
+
 ## Development standards
 
 - [SemVer](https://semver.org) is used for versioning
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		["AI2Bot","Ai2Bot-Dolma","Amazonbot","anthropic-ai","Applebot","Applebot-Extended","Bytespider","CCBot","ChatGPT-User","Claude-Web","ClaudeBot","cohere-ai","Diffbot","DuckAssistBot","FacebookBot","FriendlyCrawler","Google-Extended","GoogleOther","GoogleOther-Image","GoogleOther-Video","GPTBot","iaskspider/2.0","ICC-Crawler","ImagesiftBot","img2dataset","ISSCyberRiskCrawler","Kangaroo Bot","Meta-ExternalAgent","Meta-ExternalFetcher","OAI-SearchBot","omgili","omgilibot","PanguBot","PerplexityBot","PetalBot","Scrapy","Sidetrade indexer bot","Timpibot","VelenPublicWebCrawler","Webzio-Extended","YouBot"]