Merge pull request #121 from josephnowak/fix/error-during-append

The system permanently overwriting the data even if it was not necess…

setup.py

@@ -5,7 +5,7 @@
 
 setup(
     name="TensorDB",
-    version="0.31.0",
+    version="0.31.1",
     description="Database based in a file system storage combined with Xarray and Zarr",
     author="Joseph Nowak",
     author_email="josephgonowak97@gmail.com",

tensordb/storages/zarr_storage.py

@@ -244,6 +244,7 @@ def append(
         A list of xr.backends.ZarrStore produced by the to_zarr method executed in every dimension
 
         """
+
         if not self.exist():
             return [self.store(new_data=new_data, compute=compute)]
 
@@ -266,7 +267,7 @@ def append(
             if len(coord_to_append) == 0:
                 continue
 
-            rewrite |= ~self._validate_sorted_append(
+            rewrite |= not self._validate_sorted_append(
                 current_coord=act_coord, append_coord=coord_to_append, dim=dim
             )
 
@@ -305,6 +306,7 @@ def append(
                 continue
 
             data_to_append = complete_data.isel(**slices_to_append[dim])
+            data_to_append = data_to_append.chunk(self.chunks)
 
             delayed_appends.append(
                 data_to_append.to_zarr(

tensordb/tests/test_zarr_storage.py

@@ -1,6 +1,7 @@
 import dask
 import fsspec
 import numpy as np
+import pandas as pd
 import pytest
 import xarray as xr
 
@@ -267,12 +268,34 @@ def test_keep_sorted(self):
         arr = self.arr.chunk(index=3, columns=2)
         new_data = arr.sel(index=[3, 2, 0, 4, 1], columns=[3, 4, 2, 0, 1])
         result = self.storage_sorted_unique._keep_sorted_coords(new_data=new_data)
-        assert result.chunks == ((1, 1, 1, 1, 1), (1, 1, 1, 2))
+        assert result.chunks == ((2, 3), (2, 2, 1))
 
         new_data = arr.sel(columns=[3, 4, 2, 0, 1], index=[4, 3, 2, 1, 0])
         result = self.storage_sorted_unique._keep_sorted_coords(new_data=new_data)
         assert result.chunks == ((2, 3), (5,))
 
+    def test_insert_in_the_middle(self):
+        storage = self.storage_sorted_unique
+        arr = xr.DataArray(
+            [[1, 5], [4, 2]],
+            dims=["index", "columns"],
+            coords={"index": np.array([1, 5], "datetime64[ns]"), "columns": [1, 6]},
+        )
+        storage.store(arr)
+
+        append_arr = xr.DataArray(
+            [[100]],
+            dims=arr.dims,
+            coords={"index": np.array([3], "datetime64[ns]"), "columns": [3]},
+        )
+        storage.append(append_arr)
+
+        assert storage.read().equals(
+            arr.combine_first(append_arr).sel(
+                index=np.array([5, 3, 1], "datetime64[ns]"), columns=[6, 3, 1]
+            )
+        )
+
 
 if __name__ == "__main__":
     test = TestZarrStorage()