site: use virtualenv and add make lint (#14428)

Author: kevinjqliu

.github/workflows/docs-ci.yml

@@ -27,7 +27,10 @@ on:
 
 jobs:
   build-docs:
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest]
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5

.gitignore

@@ -29,6 +29,7 @@ site/site/
 site/docs/docs/
 site/docs/.asf.yaml
 site/docs/javadoc/
+site/.venv/
 
 # benchmark output
 spark/v3.4/spark/benchmark/*

site/Makefile

@@ -29,6 +29,14 @@ build: # Clean and build the docs site locally.
 deploy: # Clean, build, and deploy the Iceberg docs site.
 	dev/deploy.sh $(remote_name)
 
+.PHONY: lint
+lint: # Check linting on the docs.
+	dev/lint.sh
+
+.PHONY: lint-fix
+lint-fix: # Run linting with auto-fix on the docs.
+	dev/lint.sh --fix
+
 .PHONY: clean
 clean: # Clean the local docs site.
 	dev/clean.sh

site/README.md

@@ -76,6 +76,7 @@ The docs are built, run, and released using [make](https://www.gnu.org/software/
 > help: Show help for each of the Makefile recipes.
 > [serve](dev/serve.sh): Clean, build, and run the site locally.
 > [lint](dev/lint.sh): Scan markdown files for style issues.
+> [lint-fix](dev/lint.sh): Run linting with auto-fix on the markdown files.
 
 To scaffold the versioned docs and build the project, run the `build` recipe.
 
@@ -103,9 +104,18 @@ This step will generate the staged source code which blends into the original so
     └─.asf.yaml
 ```
 
-It will also scan all markdown files and fail the build on any style issues. To fix style issues, run the `lint` script with fix mode.
+#### Linting
+
+To check for markdown style issues without building the entire site, use the `lint` make command:
+
+```sh
+make lint
+```
+
+To automatically fix markdown style issues, use the `lint-fix` make command:
+
 ```sh
-./dev/lint.sh --fix
+make lint-fix
 ```
 
 <!-- markdown-link-check-disable-next-line -->

site/dev/build.sh

@@ -16,10 +16,11 @@
 # limitations under the License.
 #
 
+source dev/common.sh
 set -e
 
 ./dev/setup_env.sh
 
 ./dev/lint.sh
 
-mkdocs build
+"${VENV_DIR}/bin/python3" -m mkdocs build

site/dev/common.sh

@@ -19,6 +19,7 @@
 set -e
 
 export REMOTE="iceberg_docs"
+export VENV_DIR=".venv"
 
 # Ensures the presence of a specified remote repository for documentation.
 # If the remote doesn't exist, it adds it using the provided URL.
@@ -34,12 +35,20 @@ create_or_update_docs_remote () {
   git fetch "${REMOTE}"
 }
 
+# Creates the virtual environment if it doesn't exist.
+create_venv () {
+  if [ ! -d "${VENV_DIR}" ]; then
+    echo " --> creating virtual environment at ${VENV_DIR}"
+    python3 -m venv "${VENV_DIR}"
+  fi
+}
+
 # Installs or upgrades dependencies specified in the 'requirements.txt' file using pip.
 install_deps () {
   echo " --> install deps"
 
-  # Use pip to install or upgrade dependencies from the 'requirements.txt' file quietly
-  pip3 -q install -r requirements.txt --upgrade
+  # Use pip from venv to install or upgrade dependencies from the 'requirements.txt' file quietly
+  "${VENV_DIR}/bin/pip3" -q install -r requirements.txt --upgrade
 }
 
 # Checks if a provided argument is not empty. If empty, displays an error message and exits with a status code 1.
@@ -186,16 +195,16 @@ pull_versioned_docs () {
 
 check_markdown_files () {
   echo " --> check markdown file styles"
-  if ! python3 -m pymarkdown --config markdownlint.yml scan docs/docs/nightly/docs/*.md docs/*.md README.md
+  if ! "${VENV_DIR}/bin/python3" -m pymarkdown --config markdownlint.yml scan docs/docs/nightly/docs/*.md docs/*.md README.md
   then
-    echo "Markdown style issues found. Please run './dev/lint.sh --fix' to fix them."
+    echo "Markdown style issues found. Please run 'make lint-fix' to fix them."
     exit 1
   fi
 }
 
 fix_markdown_files () {
   echo " --> fix markdown file styles"
-  python3 -m pymarkdown --config markdownlint.yml fix docs/docs/nightly/docs/*.md docs/*.md README.md
+  "${VENV_DIR}/bin/python3" -m pymarkdown --config markdownlint.yml fix docs/docs/nightly/docs/*.md docs/*.md README.md
 }
 
 # Cleans up artifacts and temporary files generated during documentation management.

site/dev/deploy.sh

@@ -16,6 +16,7 @@
 # limitations under the License.
 #
 
+source dev/common.sh
 set -e
 
 remote_name="${1:-origin}"
@@ -24,4 +25,4 @@ remote_name="${1:-origin}"
 
 echo " --> Deploy to asf-site branch of remote repository: ${remote_name}"
 
-mkdocs gh-deploy --no-history --remote-branch=asf-site --remote-name ${remote_name}
+"${VENV_DIR}/bin/python3" -m mkdocs gh-deploy --no-history --remote-branch=asf-site --remote-name ${remote_name}

site/dev/lint.sh

@@ -20,6 +20,8 @@ source dev/common.sh
 
 set -e
 
+./dev/setup_env.sh
+
 if [[ "$1" == "--fix" ]]; then
   fix_markdown_files
 else

site/dev/serve.sh

@@ -16,10 +16,11 @@
 # limitations under the License.
 #
 
+source dev/common.sh
 set -e
 
 ./dev/setup_env.sh
 
 ./dev/lint.sh
 
-mkdocs serve --dirty --watch .
+"${VENV_DIR}/bin/python3" -m mkdocs serve --dirty --watch .

site/dev/setup_env.sh

@@ -22,6 +22,8 @@ set -e
 
 clean
 
+create_venv
+
 install_deps
 
 pull_versioned_docs