Use PyData Sphinx theme version switcher (#46936)

## Why are these changes needed?

This switches to using the PyData Sphinx theme version switcher so that
we have more control of it's location and the ordering of the versions.
- added the version switcher to the navbar
- pointed the versions to the recently added `versions.json` and the
selected version to the `version` set by read the docs
- styling to limit the height of the version switcher and allow for
scrolling to see more
- styling to suppress the RTD version switcher

<img width="646" alt="Screenshot 2024-08-02 at 10 46 45 AM"
src="https://github.com/user-attachments/assets/00836b9f-5c2b-40ed-ab70-e0175ade8d0a">


## Related issue number
Closes #46382
Closes #46189


Signed-off-by: cristianjd <cristian.j.derr@gmail.com>

doc/source/_static/css/custom.css

@@ -390,3 +390,14 @@ table.autosummary tr > td:first-child > p > a > code > span {
   color: var(--pst-color-light);
   text-decoration: underline;
 }
+
+/* Prevent the the PyData theme Version Switcher from getting too large */
+.version-switcher__menu {
+  max-height: 496px;
+  overflow: scroll;
+}
+
+/* Hide the RTD version switcher since we are using PyData theme one */
+#rtd-footer-container {
+  display: none;
+}

doc/source/conf.py

@@ -294,6 +294,7 @@ def render_svg_logo(path):
     },
     "navbar_start": ["navbar-ray-logo"],
     "navbar_end": [
+        "version-switcher",
         "navbar-icon-links",
         "navbar-anyscale",
     ],
@@ -313,6 +314,10 @@ def render_svg_logo(path):
     "navigation_depth": 4,
     "pygment_light_style": "stata-dark",
     "pygment_dark_style": "stata-dark",
+    "switcher": {
+        "json_url": "https://docs.ray.io/en/master/_static/versions.json",
+        "version_match": os.getenv("READTHEDOCS_VERSION", "master"),
+    },
 }
 
 html_context = {

doc/source/custom_directives.py

@@ -1261,6 +1261,7 @@ def pregenerate_example_rsts(
 repo_url = "https://github.com/ray-project/ray.git"
 static_dir_name = "_static"
 version_json_filename = "versions.json"
+dereference_suffix = "^{}"
 
 
 def generate_version_url(version):
@@ -1286,11 +1287,11 @@ def generate_versions_json():
         "utf-8"
     )
     # Extract release versions from tags
-    tags = re.findall(r"refs/tags/(.+?)(?:\^|\s|$)", output)
+    tags = re.findall(r"refs/tags/(.+)", output)
     for tag in tags:
-        if ray_prefix in tag:
+        if ray_prefix in tag and dereference_suffix not in tag:
             version = tag.split(ray_prefix)[1]
-            if Version(version) >= Version(min_version):
+            if version not in git_versions and Version(version) >= Version(min_version):
                 git_versions.append(version)
     git_versions.sort(key=Version, reverse=True)