Python SDK: Support multiple pydantic versions (treeverse#7672)

* Python SDK: Support multiple pydantic versions

* Fix templates

* Update clients/python/scripts/pydantic.sh

Co-authored-by: Ariel Shaqed (Scolnicov) <ariels@treeverse.io>

* Add unit test flavor with pydantic V1

* Fix script

* Fix workflow

---------

Co-authored-by: Ariel Shaqed (Scolnicov) <ariels@treeverse.io>

.github/workflows/python-wrapper-unit-tests.yaml

@@ -32,6 +32,10 @@ jobs:
   unit-tests:
     needs: paths-filter
     if: ${{ needs.paths-filter.outputs.client-change == 'true' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        pydantic_v1: [ true, false ]
     name: Unit Test Python SDK Wrapper
     runs-on: ubuntu-22.04
     steps:
@@ -43,6 +47,11 @@ jobs:
         with:
           python-version: 3.9
 
+      # When installed it will skip the installation of pydantic in the next step which will install the latest > 2
+      - name: Install pydantic V1
+        if: matrix.pydantic_v1
+        run: pip install pydantic==1.10.6
+
       - name: Install dependencies
         working-directory: ./clients/python-wrapper
         run: pip install -r requirements.txt pylint pytest pytest-md pytest-emoji

Makefile

@@ -12,6 +12,7 @@ OPENAPI_LEGACY_GENERATOR_IMAGE=openapitools/openapi-generator-cli:v5.3.0
 OPENAPI_LEGACY_GENERATOR=$(DOCKER) run --user $(UID_GID) --rm -v $(shell pwd):/mnt $(OPENAPI_LEGACY_GENERATOR_IMAGE)
 OPENAPI_GENERATOR_IMAGE=treeverse/openapi-generator-cli:v7.0.0.1
 OPENAPI_GENERATOR=$(DOCKER) run --user $(UID_GID) --rm -v $(shell pwd):/mnt $(OPENAPI_GENERATOR_IMAGE)
+PY_OPENAPI_GENERATOR=$(DOCKER) run -e PYTHON_POST_PROCESS_FILE="/mnt/clients/python/scripts/pydantic.sh" --user $(UID_GID) --rm -v $(shell pwd):/mnt $(OPENAPI_GENERATOR_IMAGE)
 
 GOLANGCI_LINT_VERSION=v1.53.3
 BUF_CLI_VERSION=v1.28.1
@@ -128,8 +129,9 @@ sdk-python-legacy: api/swagger.yml  ## Generate SDK for Python client - openapi
 sdk-python: api/swagger.yml  ## Generate SDK for Python client - openapi generator version 7.0.0
 	# remove the build folder as it also holds lakefs_sdk folder which keeps because we skip it during find
 	rm -rf clients/python/build; cd clients/python && \
-		find . -depth -name lakefs_sdk -prune -o ! \( -name Gemfile -or -name Gemfile.lock -or -name _config.yml -or -name .openapi-generator-ignore -or -name templates -or -name setup.mustache -or -name client.mustache -or -name python-codegen-config.yaml \) -delete
-	$(OPENAPI_GENERATOR) generate \
+		find . -depth -name lakefs_sdk -prune -o ! \( -name Gemfile -or -name Gemfile.lock -or -name _config.yml -or -name .openapi-generator-ignore -or -name templates -or -name setup.mustache -or -name client.mustache -or -name requirements.mustache -or -name scripts -or -name pydantic.sh -or -name python-codegen-config.yaml \) -delete
+	$(PY_OPENAPI_GENERATOR) generate \
+		--enable-post-process-file \
 		-i /mnt/$< \
 		-g python \
 		-t /mnt/clients/python/templates \

clients/python-wrapper/requirements.txt

@@ -1,6 +1,5 @@
 aenum~=3.1.15
 lakefs-sdk>=1.16.0
-pydantic >= 1.10.5, < 2
 python-dateutil~=2.8.2
 PyYAML~=6.0.1
 six~=1.16.0

clients/python-wrapper/setup.py

@@ -11,7 +11,6 @@
 
 PYTHON_REQUIRES = ">=3.9"
 REQUIRES = [
-    "pydantic >= 1.10.5, < 2",
     "setuptools == 68.2.2",
     "lakefs-sdk ~= 1.0",
     "pyyaml ~= 6.0.1",

clients/python-wrapper/tests/integration/README.md

@@ -27,11 +27,11 @@ From the clients/python-wrapper directory:
 ```sh
 python3 -m venv <your_venv_path>
 source <your_venv_path>/bin/activate
-pip install -r requirements pylint
+pip install -r requirements
 ```
 
 ## Run Tests
 
 ```sh
-pylint tests
+pytest tests
 ```

clients/python-wrapper/tests/integration/test_branch.py

@@ -1,5 +1,9 @@
 import pytest
-from pydantic import ValidationError
+
+try:
+    from pydantic.v1 import ValidationError
+except ImportError:
+    from pydantic import ValidationError
 
 import lakefs
 from lakefs.exceptions import NotFoundException, TransactionException