Add fast client-side search to Airflow documentation

Dockerfile

@@ -1235,9 +1235,13 @@ function install_airflow_when_building_images() {
     set +x
     common::install_packaging_tools
     echo
-    echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
+    echo "${COLOR_BLUE}Running 'uv pip check'${COLOR_RESET}"
     echo
-    pip check
+    # Here we should use `pip check` not `uv pip check` to detect any incompatibilities that might happen
+    # between `pip` and `uv` installations
+    # However, in the current version of `pip` there is a bug that incorrectly detects `pagefind-bin` as unsupported
+    # https://github.com/pypa/pip/issues/13709 -> once this is fixed, we should bring `pip check` back.
+    uv pip check
 }
 
 common::get_colors

Dockerfile.ci

@@ -988,9 +988,13 @@ function install_airflow_when_building_images() {
     set +x
     common::install_packaging_tools
     echo
-    echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
+    echo "${COLOR_BLUE}Running 'uv pip check'${COLOR_RESET}"
     echo
-    pip check
+    # Here we should use `pip check` not `uv pip check` to detect any incompatibilities that might happen
+    # between `pip` and `uv` installations
+    # However, in the current version of `pip` there is a bug that incorrectly detects `pagefind-bin` as unsupported
+    # https://github.com/pypa/pip/issues/13709 -> once this is fixed, we should bring `pip check` back.
+    uv pip check
 }
 
 common::get_colors

airflow-core/docs/conf.py

@@ -268,6 +268,14 @@ def add_airflow_core_exclude_patterns_to_sphinx(exclude_patterns: list[str]):
     "experimental": "This is an :ref:`experimental feature <experimental>`.",
 }
 
+# Pagefind search configuration
+pagefind_exclude_patterns = [
+    "_api/**",  # Exclude auto-generated API documentation
+    "_modules/**",  # Exclude source code modules
+    "release_notes.html",  # Exclude changelog aggregation page
+    "genindex.html",  # Exclude generated index
+]
+
 # -- Options for sphinx.ext.autodoc --------------------------------------------
 # See: https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html
 

devel-common/pyproject.toml

@@ -76,6 +76,7 @@ dependencies = [
     "rich-click>=1.7.1",
     "click>=8.1.8",
     "docutils>=0.21",
+    "pagefind[bin]",
     "sphinx-airflow-theme@https://github.com/apache/airflow-site/releases/download/0.3.0/sphinx_airflow_theme-0.3.0-py3-none-any.whl",
     "sphinx-argparse>=0.4.0",
     "sphinx-autoapi>=3",

devel-common/src/docs/utils/conf_constants.py

@@ -91,6 +91,7 @@ def get_rst_epilogue(package_version: str, airflow_core: bool) -> str:
     "redirects",
     "substitution_extensions",
     "sphinx_design",
+    "pagefind_search",
 ]
 
 SPHINX_REDOC_EXTENSIONS = [

devel-common/src/sphinx_exts/pagefind_search/README.md

@@ -0,0 +1,165 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+ -->
+
+# Pagefind Search Extension
+
+A Sphinx extension providing fast, self-hosted search for Apache Airflow documentation using [Pagefind](https://pagefind.app/).
+
+## Features
+
+- **Automatic indexing** when docs are built
+- **Cmd+K search** with modern UI and keyboard shortcuts
+- **Self-hosted** - No third-party services
+- **Content weighting** - Prioritizes titles and headings for better relevance
+- **Optimized ranking** - Tuned for exact phrase matching and title matches
+- **Playground support** - Debug and tune search behavior
+
+## Usage
+
+### Search Interface
+
+- **Keyboard**: Press `Cmd+K` (Mac) or `Ctrl+K` (Windows/Linux)
+- **Mouse**: Click the search button in the header
+- **Navigation**: Arrow keys to navigate, Enter to select, Esc to close
+
+## Configuration
+
+In Sphinx's `conf.py`:
+
+```python
+# Enable/disable search (default: True)
+pagefind_enabled = True
+
+# Verbose logging (default: False)
+pagefind_verbose = False
+
+# Content selector (default: "main")
+pagefind_root_selector = "main"
+
+# Exclude selectors (default: see below)
+# These elements won't be included in the search index
+pagefind_exclude_selectors = [
+    ".headerlink",  # Permalink icons
+    ".toctree-wrapper",  # Table of contents navigation
+    "nav",  # All navigation elements
+    "footer",  # Footer content
+    ".td-sidebar",  # Left sidebar
+    ".breadcrumb",  # Breadcrumb navigation
+    ".navbar",  # Top navigation bar
+    ".dropdown-menu",  # Dropdown menus (version selector, etc.)
+    ".docs-version-selector",  # Version selector widget
+    "[role='navigation']",  # ARIA navigation landmarks
+    ".d-print-none",  # Print-hidden elements (usually UI controls)
+    ".pagefind-search-button",  # Search button itself
+]
+
+# File pattern (default: "**/*.html")
+pagefind_glob = "**/*.html"
+
+# Exclude patterns (default: [])
+# Path patterns to exclude from indexing (e.g., auto-generated API docs)
+# Note: File-by-file indexing is used when patterns are specified (slightly slower but precise)
+# Pagefind does NOT automatically exclude underscore-prefixed directories
+pagefind_exclude_patterns = [
+    "_api/**",  # Exclude API documentation
+    "_modules/**",  # Exclude source code modules
+    "release_notes.html",  # Exclude specific files
+    "genindex.html",  # Exclude generated index
+]
+
+# Content weighting (default: True)
+# Uses lightweight regex to add data-pagefind-weight attributes to titles and headings
+pagefind_content_weighting = True
+
+# Enable playground (default: False)
+# Creates a playground at /_pagefind/playground/ for debugging search
+pagefind_enable_playground = False
+
+# Custom records for non-HTML content (default: [])
+pagefind_custom_records = [
+    {
+        "url": "/downloads/guide.pdf",
+        "content": "PDF content...",
+        "language": "en",
+        "meta": {"title": "Guide PDF"},
+    }
+]
+```
+
+### Ranking Optimization
+
+The extension uses optimized ranking parameters in `search.js`:
+
+- **termFrequency: 1.0** - Standard term occurrence weighting
+- **termSaturation: 0.7** - Moderate saturation to prevent over-rewarding repetition
+- **termSimilarity: 7.5** - Maximum boost for exact phrase matches and similar terms
+- **pageLength: 0** - No penalty for longer pages (important for reference documentation)
+
+Combined with content weighting (10x for titles, 9x for h1, 7x for h2), these settings ensure exact title matches rank highly even for very long pages.
+
+## Architecture
+
+### Build Process
+
+1. Sphinx builds HTML documentation
+2. Extension copies CSS/JS to `_static/`
+3. Extension injects search modal HTML into pages
+4. (Optional) Extension adds content weights to HTML files
+5. Extension builds Pagefind index with configured options
+
+### Runtime
+
+1. User presses Cmd+K or clicks search button
+2. JavaScript loads Pagefind library dynamically
+3. Search executes client-side
+4. Results rendered in modal
+
+## Troubleshooting
+
+### Search not working
+
+1. Check Pagefind is installed: `python -c "import pagefind; print('OK')"`
+2. Check index exists: `ls generated/_build/docs/apache-airflow/stable/_pagefind/`
+3. Enable verbose logging: `pagefind_verbose = True` in `conf.py`
+
+### Index not created
+
+- Ensure `pagefind_enabled = True` in `conf.py`
+- Check build logs for errors
+- If `pagefind[bin]` unavailable, use: `npx pagefind --site <build-dir>`
+
+### Poor search ranking
+
+1. Enable playground: `pagefind_enable_playground = True` in `conf.py`
+2. Rebuild docs and access playground at `/_pagefind/playground/`
+3. Test queries and analyze ranking scores
+4. Ensure `pagefind_content_weighting = True` (default)
+5. Check that titles and headings contain expected keywords
+
+### Debugging with Playground
+
+The playground provides detailed insights:
+
+- View all indexed pages
+- See ranking scores for each result
+- Analyze impact of different search terms
+- Verify content weighting is applied
+- Test ranking parameter changes
+
+Example, access at: `http://localhost:8000/docs/apache-airflow/stable/_pagefind/playground/`

devel-common/src/sphinx_exts/pagefind_search/__init__.py

@@ -0,0 +1,103 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Sphinx extension for Pagefind search integration."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from sphinx.application import Sphinx
+
+from sphinx_exts.pagefind_search.builder import build_index_finished, copy_static_files
+
+
+def register_templates(app: Sphinx, config) -> None:
+    """Register template directory for searchbox override."""
+    template_dir = str(Path(__file__).parent / "templates")
+    # Prepend so our template overrides Sphinx's default searchbox
+    if template_dir not in config.templates_path:
+        config.templates_path.insert(0, template_dir)
+
+
+def inject_search_html(app: Sphinx, pagename: str, templatename: str, context: dict, doctree) -> None:
+    """Inject Pagefind search modal HTML into page context."""
+    template_dir = Path(__file__).parent / "templates"
+    modal_file = template_dir / "search-modal.html"
+
+    if not modal_file.exists():
+        return
+
+    modal_content = modal_file.read_text()
+
+    from jinja2 import Template
+
+    modal_template = Template(modal_content)
+    search_modal_html = modal_template.render(pathto=context.get("pathto"))
+
+    context["pagefind_search_modal"] = search_modal_html
+
+    if "body" in context:
+        context["body"] = search_modal_html + context["body"]
+
+
+def setup(app: Sphinx) -> dict[str, Any]:
+    """Setup the Pagefind search extension."""
+    app.add_config_value("pagefind_enabled", True, "html", [bool])
+    app.add_config_value("pagefind_verbose", False, "html", [bool])
+    app.add_config_value("pagefind_root_selector", "main", "html", [str])
+    app.add_config_value(
+        "pagefind_exclude_selectors",
+        [
+            ".headerlink",
+            ".toctree-wrapper",
+            "nav",
+            "footer",
+            ".td-sidebar",
+            ".breadcrumb",
+            ".navbar",
+            ".dropdown-menu",
+            ".docs-version-selector",
+            "[role='navigation']",
+            ".d-print-none",
+            ".pagefind-search-button",
+        ],
+        "html",
+        [list],
+    )
+    app.add_config_value("pagefind_glob", "**/*.html", "html", [str])
+    app.add_config_value("pagefind_exclude_patterns", [], "html", [list])
+    app.add_config_value("pagefind_custom_records", [], "html", [list])
+    app.add_config_value("pagefind_content_weighting", True, "html", [bool])
+    app.add_config_value("pagefind_enable_playground", False, "html", [bool])
+
+    app.add_css_file("css/pagefind.css")
+    app.add_js_file("js/search.js")
+
+    # Register template directory for searchbox override
+    app.connect("config-inited", register_templates)
+    app.connect("html-page-context", inject_search_html)
+    app.connect("build-finished", copy_static_files, priority=100)
+    app.connect("build-finished", build_index_finished, priority=900)
+
+    return {
+        "version": "1.0.0",
+        "parallel_read_safe": True,
+        "parallel_write_safe": False,
+    }