OpenAPI Python Client (#1725)

.github/workflows/goreleaser.yaml

@@ -33,3 +33,4 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+

.github/workflows/pypi.yaml

@@ -0,0 +1,26 @@
+name: pypi
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  pypi:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Extract version
+        shell: bash
+        run: echo "::set-output name=tag::$(echo ${GITHUB_REF##*/} | sed s/^v//g)"
+        id: version
+      - name: Python build and make package
+        run: make package-python PACKAGE_VERSION=${{ steps.version.outputs.tag }}
+      - name: Python publish package
+        if: startsWith(github.ref, 'refs/tags')
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          user: __token__
+          password: ${{ secrets.PYPI_API_TOKEN }}
+          packages_dir: clients/python/dist/
+          verbose: true

Makefile

@@ -7,6 +7,15 @@ NPM=$(or $(shell which npm), $(error "Missing dependency - no npm in PATH"))
 PROTOC_IMAGE="treeverse/protoc:3.14.0"
 PROTOC=$(DOCKER) run --rm -v $(shell pwd):/mnt $(PROTOC_IMAGE)
 
+# https://openapi-generator.tech
+OPENAPI_GENERATOR_IMAGE=openapitools/openapi-generator-cli:v5.1.0
+OPENAPI_GENERATOR=$(DOCKER) run --rm -v $(shell pwd):/mnt $(OPENAPI_GENERATOR_IMAGE)
+ifndef PACKAGE_VERSION
+	PACKAGE_VERSION=0.1.0.dev
+endif
+
+PYTHON_IMAGE=python:3
+
 export PATH:= $(PATH):$(GOBINPATH)
 
 GOBUILD=$(GOCMD) build
@@ -91,6 +100,21 @@ go-install: go-mod-download ## Install dependencies
 	$(GOCMD) install google.golang.org/protobuf/cmd/protoc-gen-go
 
 
+client-python: api/swagger.yml  ## Generate SDK for Python client
+	$(OPENAPI_GENERATOR) generate \
+		-i /mnt/$< \
+		-g python \
+		--package-name lakefs_client \
+		--additional-properties=infoName=Treeverse,infoEmail=services@treeverse.io,packageName=lakefs_client,packageVersion=$(PACKAGE_VERSION),projectName=lakefs-client,packageUrl=https://github.com/treeverse/lakeFS/tree/master/clients/python \
+		-o /mnt/clients/python
+
+clients: client-python
+
+package-python: client-python
+	$(DOCKER) run --rm -v $(shell pwd):/mnt -w /mnt/clients/python $(PYTHON_IMAGE) ./build-package.sh
+
+package: package-python
+
 gen-api: go-install ## Run the swagger code generator
 	$(GOGENERATE) ./pkg/api
 
@@ -149,7 +173,10 @@ validate-proto: proto  ## build proto and check if diff found
 	git diff --quiet -- pkg/graveler/committed/committed.pb.go
 	git diff --quiet -- pkg/graveler/graveler.pb.go
 
-checks-validator: lint validate-fmt validate-proto  ## Run all validation/linting steps
+validate-client-python:
+	git diff --quiet -- client/python/lakefs_client/api
+
+checks-validator: lint validate-fmt validate-proto validate-client-python ## Run all validation/linting steps
 
 $(UI_DIR)/node_modules:
 	cd $(UI_DIR) && $(NPM) install
@@ -175,4 +202,4 @@ help:  ## Show Help menu
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
 # helpers
-gen: gen-api gen-ui gen-ddl gen-mockgen
+gen: gen-api gen-ui gen-ddl gen-mockgen clients

clients/python/.gitignore

@@ -0,0 +1,66 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+venv/
+.venv/
+.python-version
+.pytest_cache
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+#Ipython Notebook
+.ipynb_checkpoints

clients/python/.openapi-generator-ignore

@@ -0,0 +1,29 @@
+# OpenAPI Generator Ignore
+# Generated by openapi-generator https://github.com/openapitools/openapi-generator
+
+# Use this file to prevent files from being overwritten by the generator.
+# The patterns follow closely to .gitignore or .dockerignore.
+
+# As an example, the C# client generator defines ApiClient.cs.
+# You can make changes and tell OpenAPI Generator to ignore just this file by uncommenting the following line:
+#ApiClient.cs
+
+# You can match any string of characters against a directory, file or extension with a single asterisk (*):
+#foo/*/qux
+# The above matches foo/bar/qux and foo/baz/qux, but not foo/bar/baz/qux
+
+# You can recursively match patterns against a directory, file or extension with a double asterisk (**):
+#foo/**/qux
+# This matches foo/bar/qux, foo/baz/qux, and foo/bar/baz/qux
+
+# You can also negate patterns with an exclamation (!).
+# For example, you can ignore all files in a docs folder with the file extension .md:
+#docs/*.md
+# Then explicitly reverse the ignore rule for a single file:
+#!docs/README.md
+
+.gitlab-ci.yml
+git_push.sh
+build-package.sh
+lakefs_client/client.py
+

clients/python/.openapi-generator/FILES

@@ -0,0 +1,198 @@
+.gitignore
+.gitlab-ci.yml
+.travis.yml
+README.md
+docs/AccessKeyCredentials.md
+docs/ActionRun.md
+docs/ActionRunList.md
+docs/ActionsApi.md
+docs/AuthApi.md
+docs/BranchCreation.md
+docs/BranchesApi.md
+docs/Commit.md
+docs/CommitCreation.md
+docs/CommitList.md
+docs/CommitsApi.md
+docs/Config.md
+docs/ConfigApi.md
+docs/Credentials.md
+docs/CredentialsList.md
+docs/CredentialsWithSecret.md
+docs/CurrentUser.md
+docs/Diff.md
+docs/DiffList.md
+docs/Error.md
+docs/Group.md
+docs/GroupCreation.md
+docs/GroupList.md
+docs/HealthCheckApi.md
+docs/HookRun.md
+docs/HookRunList.md
+docs/Merge.md
+docs/MergeResult.md
+docs/MergeResultSummary.md
+docs/MetadataApi.md
+docs/ObjectStageCreation.md
+docs/ObjectStats.md
+docs/ObjectStatsList.md
+docs/ObjectsApi.md
+docs/Pagination.md
+docs/Policy.md
+docs/PolicyList.md
+docs/Ref.md
+docs/RefList.md
+docs/RefsApi.md
+docs/RefsDump.md
+docs/RepositoriesApi.md
+docs/Repository.md
+docs/RepositoryCreation.md
+docs/RepositoryList.md
+docs/ResetCreation.md
+docs/RevertCreation.md
+docs/Setup.md
+docs/StagingApi.md
+docs/StagingLocation.md
+docs/StagingMetadata.md
+docs/Statement.md
+docs/StorageURI.md
+docs/TagCreation.md
+docs/TagsApi.md
+docs/UnderlyingObjectProperties.md
+docs/User.md
+docs/UserCreation.md
+docs/UserList.md
+lakefs_client/__init__.py
+lakefs_client/api/__init__.py
+lakefs_client/api/actions_api.py
+lakefs_client/api/auth_api.py
+lakefs_client/api/branches_api.py
+lakefs_client/api/commits_api.py
+lakefs_client/api/config_api.py
+lakefs_client/api/health_check_api.py
+lakefs_client/api/metadata_api.py
+lakefs_client/api/objects_api.py
+lakefs_client/api/refs_api.py
+lakefs_client/api/repositories_api.py
+lakefs_client/api/staging_api.py
+lakefs_client/api/tags_api.py
+lakefs_client/api_client.py
+lakefs_client/apis/__init__.py
+lakefs_client/configuration.py
+lakefs_client/exceptions.py
+lakefs_client/model/__init__.py
+lakefs_client/model/access_key_credentials.py
+lakefs_client/model/action_run.py
+lakefs_client/model/action_run_list.py
+lakefs_client/model/branch_creation.py
+lakefs_client/model/commit.py
+lakefs_client/model/commit_creation.py
+lakefs_client/model/commit_list.py
+lakefs_client/model/config.py
+lakefs_client/model/credentials.py
+lakefs_client/model/credentials_list.py
+lakefs_client/model/credentials_with_secret.py
+lakefs_client/model/current_user.py
+lakefs_client/model/diff.py
+lakefs_client/model/diff_list.py
+lakefs_client/model/error.py
+lakefs_client/model/group.py
+lakefs_client/model/group_creation.py
+lakefs_client/model/group_list.py
+lakefs_client/model/hook_run.py
+lakefs_client/model/hook_run_list.py
+lakefs_client/model/merge.py
+lakefs_client/model/merge_result.py
+lakefs_client/model/merge_result_summary.py
+lakefs_client/model/object_stage_creation.py
+lakefs_client/model/object_stats.py
+lakefs_client/model/object_stats_list.py
+lakefs_client/model/pagination.py
+lakefs_client/model/policy.py
+lakefs_client/model/policy_list.py
+lakefs_client/model/ref.py
+lakefs_client/model/ref_list.py
+lakefs_client/model/refs_dump.py
+lakefs_client/model/repository.py
+lakefs_client/model/repository_creation.py
+lakefs_client/model/repository_list.py
+lakefs_client/model/reset_creation.py
+lakefs_client/model/revert_creation.py
+lakefs_client/model/setup.py
+lakefs_client/model/staging_location.py
+lakefs_client/model/staging_metadata.py
+lakefs_client/model/statement.py
+lakefs_client/model/storage_uri.py
+lakefs_client/model/tag_creation.py
+lakefs_client/model/underlying_object_properties.py
+lakefs_client/model/user.py
+lakefs_client/model/user_creation.py
+lakefs_client/model/user_list.py
+lakefs_client/model_utils.py
+lakefs_client/models/__init__.py
+lakefs_client/models/__init__.py
+lakefs_client/rest.py
+requirements.txt
+setup.cfg
+setup.py
+test-requirements.txt
+test/__init__.py
+test/test_access_key_credentials.py
+test/test_action_run.py
+test/test_action_run_list.py
+test/test_actions_api.py
+test/test_auth_api.py
+test/test_branch_creation.py
+test/test_branches_api.py
+test/test_commit.py
+test/test_commit_creation.py
+test/test_commit_list.py
+test/test_commits_api.py
+test/test_config.py
+test/test_config_api.py
+test/test_credentials.py
+test/test_credentials_list.py
+test/test_credentials_with_secret.py
+test/test_current_user.py
+test/test_diff.py
+test/test_diff_list.py
+test/test_error.py
+test/test_group.py
+test/test_group_creation.py
+test/test_group_list.py
+test/test_health_check_api.py
+test/test_hook_run.py
+test/test_hook_run_list.py
+test/test_merge.py
+test/test_merge_result.py
+test/test_merge_result_summary.py
+test/test_metadata_api.py
+test/test_object_stage_creation.py
+test/test_object_stats.py
+test/test_object_stats_list.py
+test/test_objects_api.py
+test/test_pagination.py
+test/test_policy.py
+test/test_policy_list.py
+test/test_ref.py
+test/test_ref_list.py
+test/test_refs_api.py
+test/test_refs_dump.py
+test/test_repositories_api.py
+test/test_repository.py
+test/test_repository_creation.py
+test/test_repository_list.py
+test/test_reset_creation.py
+test/test_revert_creation.py
+test/test_setup.py
+test/test_staging_api.py
+test/test_staging_location.py
+test/test_staging_metadata.py
+test/test_statement.py
+test/test_storage_uri.py
+test/test_tag_creation.py
+test/test_tags_api.py
+test/test_underlying_object_properties.py
+test/test_user.py
+test/test_user_creation.py
+test/test_user_list.py
+tox.ini

clients/python/.openapi-generator/VERSION

@@ -0,0 +1 @@
+5.1.0