local block adaptor import support (#5277)

* local block adaptor import support * apply code review changes * local cache information for import performance * fix lint errors * integration test for default import - will ignore file size as we generate locally smaller files - does not support non directory source * remove unused and reuse block type as prefix * Update docs/reference/configuration.md Co-authored-by: Ariel Shaqed (Scolnicov) <ariels@treeverse.io> * calc hash as part of iteration * using constant for magic numbers * fix lint err on md5 * Skip hidden files by default * Update import_hidden parameter --------- Co-authored-by: Ariel Shaqed (Scolnicov) <ariels@treeverse.io>
treeverse · Feb 21, 2023 · 1aedf49 · 1aedf49
1 parent a940c20
commit 1aedf49
Show file tree

Hide file tree

Showing 40 changed files with 582 additions and 134 deletions.
diff --git a/api/swagger.yml b/api/swagger.yml
@@ -79,6 +79,12 @@ components:
         application/json:
           schema:
             $ref: "#/components/schemas/Error"
+    Forbidden:
+      description: Access is forbidden
+      content:
+        application/json:
+          schema:
+            $ref: "#/components/schemas/Error"
     NotFound:
       description: Resource Not Found
       content:
@@ -937,6 +943,7 @@ components:
         - blockstore_namespace_example
         - blockstore_namespace_ValidityRegex
         - pre_sign_support
+        - import_support
       properties:
         blockstore_type:
           type: string
@@ -948,6 +955,8 @@ components:
           type: string
         pre_sign_support:
           type: boolean
+        import_support:
+          type: boolean
 
     VersionConfig:
       type: object
@@ -3150,6 +3159,8 @@ paths:
           $ref: "#/components/responses/ValidationError"
         401:
           $ref: "#/components/responses/Unauthorized"
+        403:
+          $ref: "#/components/responses/Forbidden"
         404:
           $ref: "#/components/responses/NotFound"
         default:

diff --git a/clients/java/api/openapi.yaml b/clients/java/api/openapi.yaml
diff --git a/clients/java/docs/ImportApi.md b/clients/java/docs/ImportApi.md
diff --git a/clients/java/docs/StorageConfig.md b/clients/java/docs/StorageConfig.md
diff --git a/clients/java/src/main/java/io/lakefs/clients/api/ImportApi.java b/clients/java/src/main/java/io/lakefs/clients/api/ImportApi.java
diff --git a/clients/java/src/main/java/io/lakefs/clients/api/model/StorageConfig.java b/clients/java/src/main/java/io/lakefs/clients/api/model/StorageConfig.java
diff --git a/clients/java/src/test/java/io/lakefs/clients/api/model/StorageConfigTest.java b/clients/java/src/test/java/io/lakefs/clients/api/model/StorageConfigTest.java
diff --git a/clients/python/docs/ImportApi.md b/clients/python/docs/ImportApi.md
diff --git a/clients/python/docs/StorageConfig.md b/clients/python/docs/StorageConfig.md
diff --git a/clients/python/lakefs_client/model/storage_config.py b/clients/python/lakefs_client/model/storage_config.py
diff --git a/cmd/lakefs/cmd/run.go b/cmd/lakefs/cmd/run.go
@@ -431,6 +431,7 @@ func checkMetadataPrefix(ctx context.Context, repo *catalog.Repository, logger l
 	if _, err := adapter.Get(ctx, block.ObjectPointer{
 		StorageNamespace: repo.StorageNamespace,
 		Identifier:       dummyFile,
+		IdentifierType:   block.IdentifierTypeRelative,
 	}, -1); err != nil {
 		logger.WithFields(logging.Fields{
 			"path":              dummyFile,

diff --git a/docs/assets/js/swagger.yml b/docs/assets/js/swagger.yml
@@ -79,6 +79,12 @@ components:
         application/json:
           schema:
             $ref: "#/components/schemas/Error"
+    Forbidden:
+      description: Access is forbidden
+      content:
+        application/json:
+          schema:
+            $ref: "#/components/schemas/Error"
     NotFound:
       description: Resource Not Found
       content:
@@ -937,6 +943,7 @@ components:
         - blockstore_namespace_example
         - blockstore_namespace_ValidityRegex
         - pre_sign_support
+        - import_support
       properties:
         blockstore_type:
           type: string
@@ -948,6 +955,8 @@ components:
           type: string
         pre_sign_support:
           type: boolean
+        import_support:
+          type: boolean
 
     VersionConfig:
       type: object
@@ -3150,6 +3159,8 @@ paths:
           $ref: "#/components/responses/ValidationError"
         401:
           $ref: "#/components/responses/Unauthorized"
+        403:
+          $ref: "#/components/responses/Forbidden"
         404:
           $ref: "#/components/responses/NotFound"
         default:

diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md
@@ -109,6 +109,9 @@ This reference uses `.` to denote the nesting of values.
    If specified, the storage namespace will be filled with this default value as a prefix when creating a repository from the UI.
    The user may still change it to something else.
 * `blockstore.local.path` `(string: "~/lakefs/data")` - When using the local Block Adapter, which directory to store files in
+* `blockstore.local.import_enabled` `(bool: false)` - Enable import for local Block Adapter, relevant only if you are using shared location
+* `blockstore.local.import_hidden` `(bool: false)` - When enabled import will scan and import any file or folder that starts with a dot character.
+* `blockstore.local.allowed_external_prefixes` `([]string: [])` - List of absolute path prefixes used to match any access for external location by the local block adapter (ex: /var/data/).
 * `blockstore.gs.credentials_file` `(string : )` - If specified will be used as a file path of the JSON file that contains your Google service account key
 * `blockstore.gs.credentials_json` `(string : )` - If specified will be used as JSON string that contains your Google service account key (when credentials_file is not set)
 * `blockstore.gs.pre_signed_expiry` `(time duration : "15m")` - Expiry of pre-signed URL.