Add option to disable checksums for external files and blobs

CHANGELOG.md

@@ -1,5 +1,8 @@
 ## Release notes
 
+### 0.13.6 -- TBD
+* Add - Config option to set threshold for when to stop using checksums for filepath stores. PR #1025
+
 ### 0.13.5 -- May 19, 2022
 * Update - Import ABC from collections.abc for Python 3.10 compatibility
 * Bugfix - Fix multiprocessing value error (#1013) PR #1026

datajoint/external.py

@@ -9,6 +9,9 @@
 from .declare import EXTERNAL_TABLE_ROOT
 from . import s3
 from .utils import safe_write, safe_copy
+import logging
+
+logger = logging.getLogger(__name__)
 
 CACHE_SUBFOLDING = (
     2,
@@ -310,21 +313,37 @@ def download_filepath(self, filepath_hash):
             )
             external_path = self._make_external_filepath(relative_filepath)
             local_filepath = Path(self.spec["stage"]).absolute() / relative_filepath
-            file_exists = (
-                Path(local_filepath).is_file()
-                and uuid_from_file(local_filepath) == contents_hash
+
+            file_exists = Path(local_filepath).is_file() and (
+                (
+                    config.get("filepath_checksum_size_limit")  # check if None
+                    and Path(local_filepath).stat().st_size
+                    >= config.get("filepath_checksum_size_limit")
+                )
+                or uuid_from_file(local_filepath) == contents_hash
             )
+
             if not file_exists:
                 self._download_file(external_path, local_filepath)
-                checksum = uuid_from_file(local_filepath)
-                if (
-                    checksum != contents_hash
-                ):  # this should never happen without outside interference
-                    raise DataJointError(
-                        "'{file}' downloaded but did not pass checksum'".format(
-                            file=local_filepath
+                if not config.get("filepath_checksum_size_limit") or (
+                    Path(local_filepath).stat().st_size
+                    < config.get("filepath_checksum_size_limit")
+                ):
+                    checksum = uuid_from_file(local_filepath)
+                    if (
+                        checksum != contents_hash
+                    ):  # this should never happen without outside interference
+                        raise DataJointError(
+                            "'{file}' downloaded but did not pass checksum'".format(
+                                file=local_filepath
+                            )
                         )
-                    )
+            if config.get("filepath_checksum_size_limit") and Path(
+                local_filepath
+            ).stat().st_size >= config.get("filepath_checksum_size_limit"):
+                logger.warning(
+                    f"WARNING SKIPPED CHECKSUM FOR FILE WITH HASH: {contents_hash}"
+                )
             return str(local_filepath), contents_hash
 
     # --- UTILITIES ---
@@ -402,7 +421,7 @@ def delete(
         delete_external_files=None,
         limit=None,
         display_progress=True,
-        errors_as_string=True
+        errors_as_string=True,
     ):
         """
 

datajoint/settings.py

@@ -46,6 +46,7 @@
         "display.show_tuple_count": True,
         "database.use_tls": None,
         "enable_python_native_blobs": True,  # python-native/dj0 encoding support
+        "filepath_checksum_size_limit": None,  # file size limit for when to disable checksums
     }
 )
 

docs-parts/intro/Releases_lang1.rst

@@ -1,3 +1,7 @@
+0.13.6 -- TBD
+----------------------
+* Add - Config option to set threshold for when to stop using checksums for filepath stores. PR #1025
+
 0.13.5 -- May 19, 2022
 ----------------------
 * Update - Import ABC from collections.abc for Python 3.10 compatibility

tests/test_filepath.py

@@ -3,7 +3,7 @@
 import os
 from pathlib import Path
 import random
-
+import logging
 from .schema_external import schema, Filepath, FilepathS3, stores_config
 
 
@@ -132,7 +132,9 @@ def test_duplicate_error_s3():
     test_duplicate_error(store="repo-s3")
 
 
-def test_filepath_class(table=Filepath(), store="repo"):
+def test_filepath_class(table=Filepath(), store="repo", verify_checksum=True):
+    if not verify_checksum:
+        dj.config["filepath_checksum_size_limit"] = 1  # cant be 0, breaks boolean logic
     dj.errors._switch_filepath_types(True)
     stage_path = dj.config["stores"][store]["stage"]
     # create a mock file
@@ -169,6 +171,9 @@ def test_filepath_class(table=Filepath(), store="repo"):
     # delete from external table
     table.external[store].delete(delete_external_files=True)
     dj.errors._switch_filepath_types(False)
+    dj.config["filepath_checksum_size_limit"] = None
+    # Todo: assert here that the warning from external.download_file has been logged
+    # Waiting on DJ Logging implementation
 
 
 def test_filepath_class_again():
@@ -185,6 +190,10 @@ def test_filepath_class_s3_again():
     test_filepath_class(FilepathS3(), "repo-s3")
 
 
+def test_filepath_class_no_checksum():
+    test_filepath_class(verify_checksum=False)
+
+
 def test_filepath_cleanup(table=Filepath(), store="repo"):
     """test deletion of filepath entries from external table"""