VOTE-972: Add robotstxt module. set robots.txt file per env (#758)

.gitignore

@@ -22,6 +22,7 @@
 /drush/Commands/contrib/
 /web/profiles/contrib/
 /web/libraries/
+/web/robots.txt
 
 # Ignore configuration files that may contain sensitive information.
 /web/sites/*/settings*.php

composer.json

@@ -66,6 +66,7 @@
         "drupal/redirect": "^1.8",
         "drupal/remove_http_headers": "^2.0",
         "drupal/rest_views": "^3.0",
+        "drupal/robotstxt": "^1.5",
         "drupal/s3fs": "^3.1",
         "drupal/samlauth": "^3.8",
         "drupal/simple_sitemap": "^4.1",
@@ -143,5 +144,13 @@
                 "      composer remove drupal/core-project-message"
             ]
         }
+    },
+    "scripts": {
+      "post-install-cmd": [
+        "test -e web/robots.txt && rm web/robots.txt || echo The default robots.txt file has been deleted."
+      ],
+      "post-update-cmd": [
+        "test -e web/robots.txt && rm web/robots.txt || echo The default robots.txt file has been deleted."
+      ]
     }
 }

config/production/robotstxt.settings.yml

@@ -0,0 +1,3 @@
+_core:
+  default_config_hash: ceCx5XZ_ay1Mxcv-sB95U_fBKoVkpvo8RaQiwutSZLI
+content: "#\r\n# robots.txt\r\n#\r\n# This file is to prevent the crawling and indexing of certain parts\r\n# of your site by web crawlers and spiders run by sites like Yahoo!\r\n# and Google. By telling these \"robots\" where not to go on your site,\r\n# you save bandwidth and server resources.\r\n#\r\n# This file will be ignored unless it is at the root of your host:\r\n# Used:    http://example.com/robots.txt\r\n# Ignored: http://example.com/site/robots.txt\r\n#\r\n# For more information about the robots.txt standard, see:\r\n# http://www.robotstxt.org/robotstxt.html\r\n\r\nUser-agent: *\r\nCrawl-delay: 10\r\n\r\n# Sitemaps\r\nSitemap: https://vote.gov/sitemap.xml\r\n\r\n# CSS, JS, Images\r\nAllow: /*.css$\r\nAllow: /*.js$\r\nAllow: /core/*.css$\r\nAllow: /core/*.css?\r\nAllow: /core/*.js$\r\nAllow: /modules/*.css$\r\nAllow: /modules/*.css?\r\nAllow: /modules/*.js$\r\nAllow: /modules/*.js?\r\nAllow: /modules/*.gif\r\nAllow: /modules/*.jpg\r\nAllow: /modules/*.jpeg\r\nAllow: /modules/*.png\r\nAllow: /themes/*.css$\r\nAllow: /themes/*.css?\r\nAllow: /themes/*.js$\r\nAllow: /themes/*.js?\r\nAllow: /themes/*.gif\r\nAllow: /themes/*.jpg\r\nAllow: /themes/*.jpeg\r\nAllow: /themes/*.png\r\n# Directories\r\nDisallow: /core/\r\nDisallow: /modules/\r\nDisallow: /themes/\r\n"

config/sync/config_split.config_split.production.yml

@@ -16,6 +16,7 @@ theme: {  }
 complete_list:
   - log_stdout.settings
   - new_relic_rpm.settings
+  - robotstxt.settings
   - s3fs.settings
   - samlauth.authentication
   - usagov_login.settings

config/sync/core.extension.yml

@@ -63,6 +63,7 @@ module:
   remove_http_headers: 0
   rest: 0
   rest_views: 0
+  robotstxt: 0
   serialization: 0
   simple_sitemap: 0
   system: 0

config/sync/robotstxt.settings.yml

@@ -0,0 +1 @@
+content: "#\r\n# robots.txt\r\n#\r\n# This file is to prevent the crawling and indexing of certain parts\r\n# of your site by web crawlers and spiders run by sites like Yahoo!\r\n# and Google. By telling these \"robots\" where not to go on your site,\r\n# you save bandwidth and server resources.\r\n#\r\n# This file will be ignored unless it is at the root of your host:\r\n# Used:    http://example.com/robots.txt\r\n# Ignored: http://example.com/site/robots.txt\r\n#\r\n# For more information about the robots.txt standard, see:\r\n# http://www.robotstxt.org/robotstxt.html\r\n\r\nUser-agent: *\r\nDisallow: /\r\n"

config/sync/user.role.site_builder.yml

@@ -59,6 +59,7 @@ dependencies:
     - redirect
     - remove_http_headers
     - rest
+    - robotstxt
     - simple_sitemap
     - system
     - taxonomy
@@ -110,6 +111,7 @@ permissions:
   - 'administer redirect settings'
   - 'administer redirects'
   - 'administer rest resources'
+  - 'administer robots.txt'
   - 'administer site configuration'
   - 'administer sitemap settings'
   - 'administer software updates'