Fix robots.txt to contain a public link

Replace http://backend from the robots.txt provided by the backend with
the public facing url.

Also, publish the index file instead of the single file that would be
rejected by Google.

Author: reebalazs

src/helpers/Robots/Robots.js

@@ -15,12 +15,9 @@ import { addHeadersFactory } from '@plone/volto/helpers/Proxy/Proxy';
  */
 export const generateRobots = (req) =>
   new Promise((resolve) => {
-    //const url = `${req.protocol}://${req.get('Host')}`;
-    const request = superagent.get(
-      `${
-        config.settings.internalApiPath ?? config.settings.apiPath
-      }/robots.txt`,
-    );
+    const internalUrl =
+      config.settings.internalApiPath ?? config.settings.apiPath;
+    const request = superagent.get(`${internalUrl}/robots.txt`);
     request.set('Accept', 'text/plain');
     const authToken = req.universalCookies.get('auth_token');
     if (authToken) {
@@ -31,6 +28,12 @@ export const generateRobots = (req) =>
       if (error) {
         resolve(text || error);
       } else {
+        // Plone has probably returned the sitemap link with the internal url.
+        // If so, let's replace it with the current one.
+        const url = `${req.protocol}://${req.get('Host')}`;
+        text = text.replace(internalUrl, url);
+        // Replace the sitemap with the sitemap index.
+        text = text.replace('sitemap.xml.gz', 'sitemap-index.xml');
         resolve(text);
       }
     });