docs: generate docs sitemap and robots.txt (#6855)

* docs: add sitemap to site * remove asserts from sitemap * fix asserts exclude and add user-agent to robots * add missing file * Add jekyll seo plugin to Gemfile * fix conflict * update Gemfile.lock
treeverse · Oct 25, 2023 · 804c5ee · 804c5ee
1 parent 184a926
commit 804c5ee
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 1 deletion.
diff --git a/docs/Gemfile b/docs/Gemfile
@@ -2,3 +2,5 @@ source 'https://rubygems.org'
 gem 'github-pages', group: :jekyll_plugins
 gem 'just-the-docs'
 gem "webrick", "~> 1.8"
+gem 'jekyll-sitemap'
+gem 'jekyll-seo-tag'
diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock
@@ -261,6 +261,8 @@ PLATFORMS
 
 DEPENDENCIES
   github-pages
+  jekyll-seo-tag
+  jekyll-sitemap
   just-the-docs
   webrick (~> 1.8)
 

diff --git a/docs/_config.yml b/docs/_config.yml
@@ -27,6 +27,10 @@ defaults:
       path: ""
     values:
       image: https://docs.lakefs.io/assets/img/docs_logo.png
+  - scope:
+      path: "assets"
+    values:
+      sitemap: false
 
 # general config
 repository: treeverse/lakeFS
@@ -36,6 +40,7 @@ remote_theme: pmarsceill/just-the-docs@v0.6.0
 plugins:
   - jekyll-redirect-from
   - jekyll-seo-tag
+  - jekyll-sitemap
 
 exclude: 
   - howto/deploy/includes
@@ -120,4 +125,4 @@ callouts:
   tip:
     color: green
   fubar:
-    color: purple
+    color: purple
diff --git a/docs/robots.txt b/docs/robots.txt
@@ -0,0 +1,6 @@
+---
+layout: null
+sitemap: false
+---
+User-agent: *
+Sitemap: {{ site.url }}/sitemap.xml