Merge branch 'treeverse:master' into master

.github/workflows/compatibility-tests.yaml

@@ -99,7 +99,7 @@ jobs:
         uses: docker/setup-buildx-action@v2
 
       - name: Build and export
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         id: build_export
         with:
           context: .

.github/workflows/docker-publish-exp-image.yaml

@@ -43,7 +43,9 @@ jobs:
           go-version: "1.21.4"
         id: go
 
-      - uses: bufbuild/[email protected]
+      - uses: bufbuild/[email protected]
+        with:
+          github_token: ${{ github.token }}
 
       - name: Extract version
         shell: bash
@@ -73,7 +75,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Build and push lakefs
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           target: ${{ inputs.target }}

.github/workflows/docker-publish-lakefs-rclone-export.yaml

@@ -51,7 +51,7 @@ jobs:
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Build and push image
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: deployments/tools/export
           push: true

.github/workflows/docker-publish.yaml

@@ -33,7 +33,9 @@ jobs:
           go-version: "1.21.4"
         id: go
 
-      - uses: bufbuild/[email protected]
+      - uses: bufbuild/[email protected]
+        with:
+          github_token: ${{ github.token }}
 
       - name: Extract version
         shell: bash
@@ -99,7 +101,7 @@ jobs:
             type=semver,pattern={{major}}.{{minor}},value=${{ steps.version.outputs.tag }}
             type=semver,pattern={{major}},value=${{ steps.version.outputs.tag }}
       - name: Build and push lakectl
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           target: lakectl
@@ -109,7 +111,7 @@ jobs:
           tags: ${{ steps.lakectl_meta.outputs.tags }}
 
       - name: Build and push lakefs
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v5
         with:
           context: .
           target: lakefs-plugins

.github/workflows/esti.yaml

@@ -100,7 +100,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       - name: Build and Push
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           push: true
           tags: ${{ steps.login-ecr.outputs.registry }}/lakefs:${{ steps.version.outputs.tag }}
@@ -257,7 +257,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       - name: Build and Push to Amazon ECR
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           push: true
           tags: ${{ steps.login-ecr.outputs.registry }}/lakefs-rclone-export:${{ steps.version.outputs.tag }}
@@ -583,7 +583,7 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
       - name: Build and Push hive-metastore
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           push: true
           tags: ghcr.io/treeverse/hive-metastore:${{ needs.deploy-image.outputs.tag }}

.github/workflows/python-wrapper-unit-tests.yaml

@@ -4,10 +4,34 @@ on:
     branches:
       - master
   pull_request:
-    paths:
-      - "clients/python-wrapper/**"
+
 jobs:
+  paths-filter:
+    runs-on: ubuntu-latest
+    outputs:
+      client-change: ${{ steps.filter.outputs.client == 'true' }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          filters: |
+            client: 
+              - 'clients/python-wrapper/**'
+              - '.github/workflows/python-wrapper-unit-tests.yaml'
+
+  always-succeed:
+    needs: paths-filter
+    if: ${{ needs.paths-filter.outputs.client-change != 'true' }}
+    name: Unit Test Python SDK Wrapper
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Always Succeed
+        run: echo "No changes in client - skipping unit tests"
+
   unit-tests:
+    needs: paths-filter
+    if: ${{ needs.paths-filter.outputs.client-change == 'true' }}
     name: Unit Test Python SDK Wrapper
     runs-on: ubuntu-22.04
     steps:

.github/workflows/test.yaml

@@ -20,7 +20,9 @@ jobs:
           node-version: "18.17.0"
           cache: "npm"
           cache-dependency-path: webui/package-lock.json
-      - uses: bufbuild/[email protected]
+      - uses: bufbuild/[email protected]
+        with:
+          github_token: ${{ github.token }}
       - name: Generate code
         env:
           NODE_OPTIONS: "--max-old-space-size=4096"

Makefile

@@ -14,7 +14,7 @@ OPENAPI_GENERATOR_IMAGE=treeverse/openapi-generator-cli:v7.0.0.1
 OPENAPI_GENERATOR=$(DOCKER) run --user $(UID_GID) --rm -v $(shell pwd):/mnt $(OPENAPI_GENERATOR_IMAGE)
 
 GOLANGCI_LINT_VERSION=v1.53.3
-BUF_CLI_VERSION=v1.27.1
+BUF_CLI_VERSION=v1.28.1
 
 ifndef PACKAGE_VERSION
 	PACKAGE_VERSION=0.1.0-SNAPSHOT
@@ -300,8 +300,8 @@ python-wrapper-lint:
 	$(DOCKER) run --user $(UID_GID) --rm -v $(shell pwd):/mnt -e HOME=/tmp/ -w /mnt/clients/python-wrapper $(PYTHON_IMAGE) /bin/bash -c "./pylint.sh"
 
 python-wrapper-gen-docs:
-	sphinx-build -b html clients/python-wrapper/docs clients/python-wrapper/_site/
-	sphinx-build -b html clients/python-wrapper/docs clients/python-wrapper/_site/$$(python clients/python-wrapper/setup.py --version)
+	sphinx-build -b html -W clients/python-wrapper/docs clients/python-wrapper/_site/
+	sphinx-build -b html -W clients/python-wrapper/docs clients/python-wrapper/_site/$$(python clients/python-wrapper/setup.py --version)
 
 $(UI_DIR)/node_modules:
 	cd $(UI_DIR) && $(NPM) install

clients/python-wrapper/docs/lakefs.import_manager.rst

@@ -0,0 +1,7 @@
+lakefs.import\_manager module
+=============================
+
+.. automodule:: lakefs.import_manager
+   :members:
+   :undoc-members:
+   :show-inheritance:

clients/python-wrapper/docs/lakefs.models.rst

@@ -0,0 +1,7 @@
+lakefs.models module
+====================
+
+.. automodule:: lakefs.models
+   :members:
+   :undoc-members:
+   :show-inheritance:

clients/python-wrapper/docs/lakefs.rst

@@ -8,17 +8,17 @@ Submodules
    :maxdepth: 4
 
    lakefs.branch
-   lakefs.branch_manager
    lakefs.client
    lakefs.config
    lakefs.exceptions
+   lakefs.import_manager
+   lakefs.models
+   lakefs.namedtuple
    lakefs.object
-   lakefs.object_io
    lakefs.object_manager
    lakefs.reference
    lakefs.repository
    lakefs.tag
-   lakefs.tag_manager
 
 Module contents
 ---------------

clients/python-wrapper/lakefs/.pylintrc

@@ -8,6 +8,8 @@ max-line-length=120
 max-locals=25
 # Maximum number of arguments for function / method
 max-args=10
+# Maximum number of class attributes
+max-attributes=10
 
 [MESSAGES CONTROL]
 disable=too-few-public-methods, fixme

clients/python-wrapper/lakefs/__init__.py

@@ -2,8 +2,9 @@
 Allow importing of models from package root
 """
 
-from lakefs.repository import Repository, RepositoryProperties
-from lakefs.reference import Reference, Commit, Change
+from lakefs.repository import Repository
+from lakefs.reference import Reference
+from lakefs.models import Commit, Change, RepositoryProperties
 from lakefs.tag import Tag
 from lakefs.branch import Branch
 from lakefs.object import StoredObject, WriteableObject, ObjectReader

clients/python-wrapper/lakefs/branch.py

@@ -1,6 +1,7 @@
 """
 Module containing lakeFS branch implementation
 """
+
 from __future__ import annotations
 
 from typing import Optional, Generator, Iterable, Literal
@@ -10,7 +11,8 @@
 from lakefs.object import WriteableObject
 from lakefs.object import StoredObject
 from lakefs.import_manager import ImportManager
-from lakefs.reference import Reference, Change, generate_listing
+from lakefs.reference import Reference, generate_listing
+from lakefs.models import Change
 from lakefs.exceptions import api_exception_handler, ConflictException, LakeFSException
 
 

clients/python-wrapper/lakefs/client.py

@@ -18,26 +18,12 @@
 
 from lakefs.config import ClientConfig
 from lakefs.exceptions import NoAuthenticationFound, NotAuthorizedException, ServerException
-from lakefs.namedtuple import LenientNamedTuple
+from lakefs.models import ServerStorageConfiguration
 
 # global default client
 DEFAULT_CLIENT: Optional[Client] = None
 
 
-class ServerStorageConfiguration(LenientNamedTuple):
-    """
-    Represent a lakeFS server's storage configuration
-    """
-    blockstore_type: str
-    pre_sign_support: bool
-    import_support: bool
-    blockstore_namespace_example: str
-    blockstore_namespace_validity_regex: str
-    pre_sign_support_ui: bool
-    import_validity_regex: str
-    default_namespace_prefix: Optional[str] = None
-
-
 class ServerConfiguration:
     """
     Represent a lakeFS server's configuration

clients/python-wrapper/lakefs/config.py

@@ -2,6 +2,8 @@
 Client configuration module
 """
 
+from __future__ import annotations
+
 import os
 from pathlib import Path
 

clients/python-wrapper/lakefs/exceptions.py

@@ -13,6 +13,12 @@ class LakeFSException(Exception):
     """
     Base exception for all SDK exceptions
     """
+
+
+class ServerException(LakeFSException):
+    """
+    Generic exception when no other exception is applicable
+    """
     status_code: int
     reason: str
 
@@ -21,7 +27,7 @@ def __init__(self, status=None, reason=None):
         self.message = reason
 
 
-class NotFoundException(LakeFSException):
+class NotFoundException(ServerException):
     """
     Resource could not be found on lakeFS server
     """
@@ -32,37 +38,31 @@ def __init__(self, status=None, reason=None):
         super().__init__(status, reason)
 
 
-class ForbiddenException(LakeFSException):
+class ForbiddenException(ServerException):
     """
     Operation not permitted
     """
 
 
-class NoAuthenticationFound(LakeFSException):
+class NoAuthenticationFound(ServerException):
     """
     Raised when no authentication method could be found on Client instantiation
     """
 
 
-class NotAuthorizedException(LakeFSException):
+class NotAuthorizedException(ServerException):
     """
     User not authorized to perform operation
     """
 
 
-class ServerException(LakeFSException):
-    """
-    Generic exception when no other exception is applicable
-    """
-
-
-class UnsupportedOperationException(LakeFSException):
+class UnsupportedOperationException(ServerException):
     """
     Operation not supported by lakeFS server or SDK
     """
 
 
-class ConflictException(LakeFSException):
+class ConflictException(ServerException):
     """
      Resource / request conflict
     """
@@ -74,7 +74,7 @@ class ObjectNotFoundException(NotFoundException, FileNotFoundError):
     """
 
 
-class ObjectExistsException(LakeFSException, FileExistsError):
+class ObjectExistsException(ServerException, FileExistsError):
     """
     Raised when Object('...').create(mode='x') and object exists
     """
@@ -86,12 +86,18 @@ class PermissionException(NotAuthorizedException, PermissionError):
     """
 
 
-class InvalidRangeException(LakeFSException, OSError):
+class InvalidRangeException(ServerException, OSError):
     """
     Raised when the reference could not be found in the lakeFS server
     """
 
 
+class ImportManagerException(LakeFSException):
+    """
+    Import manager exceptions that are not originated from the SDK
+    """
+
+
 _STATUS_CODE_TO_EXCEPTION = {
     http.HTTPStatus.UNAUTHORIZED.value: NotAuthorizedException,
     http.HTTPStatus.FORBIDDEN.value: ForbiddenException,