Update Makefile for python client with auto setup (#1995)

Automate python client setup and use a virtual env instead to avoid change an end-users' OS python

Author: MonkeyCanCode

.github/workflows/python-client.yml

@@ -58,15 +58,6 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install Poetry
-        run: |
-          pip install --user --upgrade -r regtests/requirements.txt
-
-      # TODO: add cache for poetry dependencies once we have poetry.lock in the repo
-      - name: Install dependencies
-        working-directory: client/python
-        run: poetry install --all-extras
-
       - name: Lint
         working-directory: client/python
         run: |
@@ -75,15 +66,14 @@ jobs:
       - name: Generated Client Tests
         working-directory: client/python
         run: |
-          export SCRIPT_DIR="non-existing-mock-directory"
-          poetry run pytest test/
+          make test-client
 
       - name: Image build
         run: |
           ./gradlew \
-              :polaris-server:assemble \
-              :polaris-server:quarkusAppPartsBuild --rerun \
-              -Dquarkus.container-image.build=true
+            :polaris-server:assemble \
+            :polaris-server:quarkusAppPartsBuild --rerun \
+            -Dquarkus.container-image.build=true
 
       - name: Integration Tests
         working-directory: client/python

client/python/.pre-commit-config.yaml

@@ -23,14 +23,17 @@ repos:
       - id: end-of-file-fixer
       - id: debug-statements
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.11.13
+    rev: v0.12.1
     hooks:
-      - id: ruff
+      # Run the linter.
+      - id: ruff-check
         args: [ --fix, --exit-non-zero-on-fix ]
+      # Run the formatter.
       - id: ruff-format
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v1.16.0
     hooks:
       - id: mypy
         args:
           [--disallow-untyped-defs, --ignore-missing-imports, --install-types, --non-interactive]
+        files: 'integration_tests/.*\.py'

client/python/Makefile

@@ -15,10 +15,65 @@
 #  specific language governing permissions and limitations
 #  under the License.
 
-regenerate-client:
+# .SILENT:
+
+# Configures the shell for recipes to use bash, enabling bash commands and ensuring
+# that recipes exit on any command failure (including within pipes).
+SHELL = /usr/bin/env bash -o pipefail
+.SHELLFLAGS = -ec
+
+# Version information
+VERSION ?= $(shell cat pyproject.toml | grep version | sed 's/version *= *"\(.*\)"/\1/')
+BUILD_DATE := $(shell date -u +"%Y-%m-%dT%H:%M:%S%:z")
+GIT_COMMIT := $(shell git rev-parse HEAD)
+POETRY_VERSION := $(shell cat pyproject.toml | grep requires-poetry | sed 's/requires-poetry *= *"\(.*\)"/\1/')
+
+# Variables
+VENV_DIR := .venv
+
+.PHONY: help
+help: ## Display this help.
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-30s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
+
+.PHONY: version
+version: ## Print version information.
+	@echo "Apache Polaris version: ${VERSION}"
+	@echo "Build date: ${BUILD_DATE}"
+	@echo "Git commit: ${GIT_COMMIT}"
+	@echo "Poetry version: ${POETRY_VERSION}"
+
+# Target to create the virtual environment directory
+$(VENV_DIR):
+	@echo "Setting up Python virtual environment at $(VENV_DIR)..."
+	python3 -m venv $(VENV_DIR)
+	@echo "Virtual environment created."
+
+.PHONY: setup-env
+setup-env: $(VENV_DIR) install-poetry-deps
+
+.PHONY: install-poetry-deps
+install-poetry-deps:
+	@echo "Installing Poetry and project dependencies into $(VENV_DIR)..."
+	# Ensure pip is up-to-date within the venv
+	$(VENV_DIR)/bin/pip install --upgrade pip
+	# Install poetry if not already present
+	@if [ ! -f "$(VENV_DIR)/bin/poetry" ]; then \
+		$(VENV_DIR)/bin/pip install --upgrade "poetry${POETRY_VERSION}"; \
+	fi
+	# Install needed dependencies using poetry
+	$(VENV_DIR)/bin/poetry install --all-extras
+	@echo "Poetry and dependencies installed."
+
+.PHONY: regenerate-client
+regenerate-client: ## Regenerate the client code
 	../templates/regenerate.sh
 
-test-integration:
+.PHONY: test-client
+test-client: setup-env ## Run client tests
+	SCRIPT_DIR="non-existing-mock-directory" $(VENV_DIR)/bin/poetry run pytest test/
+
+.PHONY: test-integration
+test-integration: setup-env ## Run integration tests
 	docker compose -f docker-compose.yml kill
 	docker compose -f docker-compose.yml rm -f
 	docker compose -f docker-compose.yml up -d
@@ -28,8 +83,25 @@ test-integration:
 		echo "Still waiting for HTTP 200 from /q/health..."; \
 	done
 	@echo "Polaris is healthy. Starting integration tests..."
-	poetry run pytest integration_tests/ ${PYTEST_ARGS}
+	$(VENV_DIR)/bin/poetry run pytest integration_tests/ ${PYTEST_ARGS}
+
+.PHONY: lint
+lint: setup-env ## Run linting checks
+	$(VENV_DIR)/bin/poetry run pre-commit run --files integration_tests/* cli/*
 
+.PHONY: clean-venv
+clean-venv:
+	@echo "Attempting to remove virtual environment directory: $(VENV_DIR)..."
+	# SAFETY CHECK: Ensure VENV_DIR is not empty and exists before attempting to remove
+	@if [ -n "$(VENV_DIR)" ] && [ -d "$(VENV_DIR)" ]; then \
+		rm -rf "$(VENV_DIR)"; \
+		echo "Virtual environment removed."; \
+	else \
+		echo "Virtual environment directory '$(VENV_DIR)' not found or VENV_DIR is empty. No action taken."; \
+	fi
 
-lint:
-	poetry run pre-commit run --files integration_tests/*
+.PHONY: clean
+clean: clean-venv ## Cleanup
+	@echo "Cleaning up Python cache files..."
+	find . -type f -name "*.pyc" -delete
+	find . -type d -name "__pycache__" -delete

client/python/README.md

@@ -6,9 +6,9 @@
   to you under the Apache License, Version 2.0 (the
   "License"); you may not use this file except in compliance
   with the License.  You may obtain a copy of the License at
- 
+
    http://www.apache.org/licenses/LICENSE-2.0
- 
+
   Unless required by applicable law or agreed to in writing,
   software distributed under the License is distributed on an
   "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,17 +24,13 @@ The Apache Polaris Python package provides a client for interacting with the Apa
 
 ### Prerequisites
 - Python 3.9 or later
-- poetry >= 2.0
+- poetry >= 2.1
 
 ### Installation
-First we need to generate the OpenAPI client code from the OpenAPI specification. 
+First we need to generate the OpenAPI client code from the OpenAPI specification.
 ```
 make regenerate-client
 ```
-Install the project with test dependencies:
-```
-poetry install --all-extras
-```
 
 ### Auto-formatting and Linting
 ```
@@ -44,4 +40,4 @@ make lint
 ### Running Integration Tests
 ```
 make test-integration
-```
+```