Merge remote-tracking branch 'origin/master' into fix/handle-hook-err…

…ors-7075

.github/workflows/compatibility-tests.yaml

@@ -99,7 +99,7 @@ jobs:
         uses: docker/setup-buildx-action@v2
 
       - name: Build and export
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         id: build_export
         with:
           context: .
@@ -121,7 +121,7 @@ jobs:
       matrix:
         # Removing a version from this list means the published client is no longer compatible with
         # that lakeFS version.
-        lakefs_version: [ 0.108.0, 0.109.0, 0.110.0, 0.111.0, 0.111.1, 0.112.1, 0.113.0, 1.0.0, 1.1.0 ]
+        lakefs_version: [ 0.108.0, 0.109.0, 0.110.0, 0.111.0, 0.111.1, 0.112.1, 0.113.0, 1.0.0, 1.1.0, 1.2.0, 1.3.0 ]
     runs-on: ubuntu-20.04
     env:
       TAG: ${{ matrix.lakefs_version }}

.github/workflows/docker-publish-exp-image.yaml

@@ -43,7 +43,9 @@ jobs:
           go-version: "1.21.4"
         id: go
 
-      - uses: bufbuild/buf-setup-action@v1.27.1
+      - uses: bufbuild/buf-setup-action@v1.28.1
+        with:
+          github_token: ${{ github.token }}
 
       - name: Extract version
         shell: bash
@@ -73,7 +75,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Build and push lakefs
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           target: ${{ inputs.target }}

.github/workflows/docker-publish-lakefs-rclone-export.yaml

@@ -51,7 +51,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Build and push image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: deployments/tools/export
           push: true

.github/workflows/docker-publish.yaml

@@ -33,7 +33,9 @@ jobs:
           go-version: "1.21.4"
         id: go
 
-      - uses: bufbuild/buf-setup-action@v1.27.1
+      - uses: bufbuild/buf-setup-action@v1.28.1
+        with:
+          github_token: ${{ github.token }}
 
       - name: Extract version
         shell: bash
@@ -99,7 +101,7 @@ jobs:
             type=semver,pattern={{major}}.{{minor}},value=${{ steps.version.outputs.tag }}
             type=semver,pattern={{major}},value=${{ steps.version.outputs.tag }}
       - name: Build and push lakectl
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           target: lakectl
@@ -109,7 +111,7 @@ jobs:
           tags: ${{ steps.lakectl_meta.outputs.tags }}
 
       - name: Build and push lakefs
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           target: lakefs-plugins

.github/workflows/docs-pr.yaml

@@ -30,11 +30,6 @@ jobs:
         working-directory: docs
         run: bundle exec jekyll build --config _config.yml -d _site/ --verbose
 
-      - name: Update robots.txt
-        working-directory: docs/_site
-        run: |
-          sed -i 's/^Sitemap: /Sitemap: https:\/\/docs.lakefs.io/' robots.txt
-
       - name: Overlay PR message on each page
         working-directory: docs/_site
         run: |

.github/workflows/docs-release.yaml

@@ -37,10 +37,10 @@ jobs:
         working-directory: docs
         run: bundle exec jekyll build --config _config.yml -d _site/${{ steps.version.outputs.tag }} -b /${{ steps.version.outputs.tag }}
 
-      - name: Update robots.txt
-        working-directory: docs/_site
+      - name: Remove robots.txt
+        working-directory: docs/_site/${{ steps.version.outputs.tag }}
         run: |
-          sed -i 's/^Sitemap: /Sitemap: https:\/\/docs.lakefs.io/' robots.txt
+          if [[ -f robots.txt ]]; then rm robots.txt; fi
 
       - name: Update published version
         env:

.github/workflows/esti.yaml

@@ -100,7 +100,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       - name: Build and Push
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           push: true
           tags: ${{ steps.login-ecr.outputs.registry }}/lakefs:${{ steps.version.outputs.tag }}
@@ -257,7 +257,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       - name: Build and Push to Amazon ECR
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           push: true
           tags: ${{ steps.login-ecr.outputs.registry }}/lakefs-rclone-export:${{ steps.version.outputs.tag }}
@@ -583,7 +583,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       - name: Build and Push hive-metastore
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           push: true
           tags: ghcr.io/treeverse/hive-metastore:${{ needs.deploy-image.outputs.tag }}

.github/workflows/python-wrapper-unit-tests.yaml

@@ -4,10 +4,34 @@ on:
     branches:
       - master
   pull_request:
-    paths:
-      - "clients/python-wrapper/**"
+
 jobs:
+  paths-filter:
+    runs-on: ubuntu-latest
+    outputs:
+      client-change: ${{ steps.filter.outputs.client == 'true' }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            client: 
+              - 'clients/python-wrapper/**'
+              - '.github/workflows/python-wrapper-unit-tests.yaml'
+
+  always-succeed:
+    needs: paths-filter
+    if: ${{ needs.paths-filter.outputs.client-change != 'true' }}
+    name: Unit Test Python SDK Wrapper
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Always Succeed
+        run: echo "No changes in client - skipping unit tests"
+
   unit-tests:
+    needs: paths-filter
+    if: ${{ needs.paths-filter.outputs.client-change == 'true' }}
     name: Unit Test Python SDK Wrapper
     runs-on: ubuntu-22.04
     steps:

.github/workflows/test.yaml

@@ -20,7 +20,9 @@ jobs:
           node-version: "18.17.0"
           cache: "npm"
           cache-dependency-path: webui/package-lock.json
-      - uses: bufbuild/buf-setup-action@v1.27.1
+      - uses: bufbuild/buf-setup-action@v1.28.1
+        with:
+          github_token: ${{ github.token }}
       - name: Generate code
         env:
           NODE_OPTIONS: "--max-old-space-size=4096"

CHANGELOG.md

@@ -1,5 +1,20 @@
 # Changelog
 
+# v1.3.0
+
+:new: What's new:
+
+- Update AWS Go SDK v2 to support S3 express directory buckets (#7083)
+- API and lakectl: support long-running dump and restore operations (#6975)
+- lakectl: Improved performance for fs rm recursive command (#7035)
+- Glue Exporter: Support Hadoop directory markers (#7058)
+
+:bug: Bugs fixed:
+
+- API: Fix StatsObject returned metadata on empty metadata (#7026)
+- API: Return the right default pagination per page- 100 instead of 1000 (#7051)
+- DynamoDB KV: Check if a table exists before we try to create it (#7056)
+
 # v1.2.0
 
 :new: What's new:

Makefile

@@ -14,7 +14,7 @@ OPENAPI_GENERATOR_IMAGE=treeverse/openapi-generator-cli:v7.0.0.1
 OPENAPI_GENERATOR=$(DOCKER) run --user $(UID_GID) --rm -v $(shell pwd):/mnt $(OPENAPI_GENERATOR_IMAGE)
 
 GOLANGCI_LINT_VERSION=v1.53.3
-BUF_CLI_VERSION=v1.27.1
+BUF_CLI_VERSION=v1.28.1
 
 ifndef PACKAGE_VERSION
 	PACKAGE_VERSION=0.1.0-SNAPSHOT
@@ -300,8 +300,8 @@ python-wrapper-lint:
 	$(DOCKER) run --user $(UID_GID) --rm -v $(shell pwd):/mnt -e HOME=/tmp/ -w /mnt/clients/python-wrapper $(PYTHON_IMAGE) /bin/bash -c "./pylint.sh"
 
 python-wrapper-gen-docs:
-	sphinx-build -b html clients/python-wrapper/docs clients/python-wrapper/_site/
-	sphinx-build -b html clients/python-wrapper/docs clients/python-wrapper/_site/$$(python clients/python-wrapper/setup.py --version)
+	sphinx-build -b html -W clients/python-wrapper/docs clients/python-wrapper/_site/
+	sphinx-build -b html -W clients/python-wrapper/docs clients/python-wrapper/_site/$$(python clients/python-wrapper/setup.py --version)
 
 $(UI_DIR)/node_modules:
 	cd $(UI_DIR) && $(NPM) install

clients/python-wrapper/docs/lakefs.branch_manager.rst → clients/python-wrapper/docs/lakefs.import_manager.rst

@@ -1,7 +1,7 @@
-lakefs.branch\_manager module
+lakefs.import\_manager module
 =============================
 
-.. automodule:: lakefs.branch_manager
+.. automodule:: lakefs.import_manager
    :members:
    :undoc-members:
    :show-inheritance:

clients/python-wrapper/docs/lakefs.models.rst

@@ -0,0 +1,7 @@
+lakefs.models module
+====================
+
+.. automodule:: lakefs.models
+   :members:
+   :undoc-members:
+   :show-inheritance:

clients/python-wrapper/docs/lakefs.rst

@@ -8,17 +8,17 @@ Submodules
    :maxdepth: 4
 
    lakefs.branch
-   lakefs.branch_manager
    lakefs.client
    lakefs.config
    lakefs.exceptions
+   lakefs.import_manager
+   lakefs.models
+   lakefs.namedtuple
    lakefs.object
-   lakefs.object_io
    lakefs.object_manager
    lakefs.reference
    lakefs.repository
    lakefs.tag
-   lakefs.tag_manager
 
 Module contents
 ---------------

clients/python-wrapper/lakefs/.pylintrc

@@ -8,6 +8,8 @@ max-line-length=120
 max-locals=25
 # Maximum number of arguments for function / method
 max-args=10
+# Maximum number of class attributes
+max-attributes=10
 
 [MESSAGES CONTROL]
 disable=too-few-public-methods, fixme

clients/python-wrapper/lakefs/__init__.py

@@ -2,11 +2,10 @@
 Allow importing of models from package root
 """
 
-from lakefs.repository import Repository, RepositoryProperties
-from lakefs.reference import Reference, Commit, Change
+from lakefs.repository import Repository
+from lakefs.reference import Reference
+from lakefs.models import Commit, Change, RepositoryProperties
 from lakefs.tag import Tag
 from lakefs.branch import Branch
 from lakefs.object import StoredObject, WriteableObject, ObjectReader
-from lakefs.branch_manager import BranchManager
 from lakefs.object_manager import ObjectManager
-from lakefs.tag_manager import TagManager

clients/python-wrapper/lakefs/branch.py

@@ -1,6 +1,7 @@
 """
 Module containing lakeFS branch implementation
 """
+
 from __future__ import annotations
 
 from typing import Optional, Generator, Iterable, Literal
@@ -10,7 +11,8 @@
 from lakefs.object import WriteableObject
 from lakefs.object import StoredObject
 from lakefs.import_manager import ImportManager
-from lakefs.reference import Reference, Change
+from lakefs.reference import Reference, generate_listing
+from lakefs.models import Change
 from lakefs.exceptions import api_exception_handler, ConflictException, LakeFSException
 
 
@@ -139,15 +141,16 @@ def uncommitted(self, max_amount: Optional[int], after: Optional[str] = None, pr
         :param max_amount: Stop showing changes after this amount
         :param after: Return items after this value
         :param prefix: Return items prefixed with this value
+        :param kwargs: Additional Keyword Arguments to send to the server
         :raises:
             NotFoundException if branch or repository do not exist
             NotAuthorizedException if user is not authorized to perform this operation
             ServerException for any other errors
         """
 
-        for diff in self._get_generator(self._client.sdk_client.branches_api.diff_branch,
-                                        self._repo_id, self._id, max_amount=max_amount, after=after, prefix=prefix,
-                                        **kwargs):
+        for diff in generate_listing(self._client.sdk_client.branches_api.diff_branch,
+                                     self._repo_id, self._id, max_amount=max_amount, after=after, prefix=prefix,
+                                     **kwargs):
             yield Change(**diff.dict())
 
     def import_data(self, commit_message: str, metadata: Optional[dict] = None) -> ImportManager:

clients/python-wrapper/lakefs/client.py

@@ -18,26 +18,12 @@
 
 from lakefs.config import ClientConfig
 from lakefs.exceptions import NoAuthenticationFound, NotAuthorizedException, ServerException
-from lakefs.namedtuple import LenientNamedTuple
+from lakefs.models import ServerStorageConfiguration
 
 # global default client
 DEFAULT_CLIENT: Optional[Client] = None
 
 
-class ServerStorageConfiguration(LenientNamedTuple):
-    """
-    Represent a lakeFS server's storage configuration
-    """
-    blockstore_type: str
-    pre_sign_support: bool
-    import_support: bool
-    blockstore_namespace_example: str
-    blockstore_namespace_validity_regex: str
-    pre_sign_support_ui: bool
-    import_validity_regex: str
-    default_namespace_prefix: Optional[str] = None
-
-
 class ServerConfiguration:
     """
     Represent a lakeFS server's configuration

clients/python-wrapper/lakefs/config.py

@@ -2,6 +2,8 @@
 Client configuration module
 """
 
+from __future__ import annotations
+
 import os
 from pathlib import Path