Merge pull request #75 from smartondev/more-test

bug: gmail restore without labelIds key in metadata (is valid from server)
test: add more tests

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## DEV
 
+- Bug: fix gmail restore without labelIds in stored message metadata
 
 ## 0.11.0
 

gwbackupy/filters/filter_interface.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from datetime import datetime
 from builtins import bool
 
@@ -8,13 +10,16 @@ class FilterInterface:
     def __init__(self):
         pass
 
-    def date_from(self, dt: datetime):
+    def with_date_from(self, dt: datetime | None):
+        pass
+
+    def with_date_to(self, dt: datetime | None):
         pass
 
-    def date_to(self, dt: datetime):
+    def with_match_deleted(self, match_deleted: bool = True):
         pass
 
-    def is_deleted(self):
+    def is_match_deleted(self) -> bool:
         pass
 
     def match(self, d: any) -> bool:

gwbackupy/filters/gmail_filter.py

@@ -1,8 +1,8 @@
+from __future__ import annotations
+
 from datetime import datetime, timezone
 from typing import Union
 
-import tzlocal
-
 from gwbackupy.filters.filter_interface import FilterInterface
 from gwbackupy.storage.storage_interface import LinkInterface
 
@@ -15,20 +15,35 @@ def __init__(self):
         self.__is_deleted: bool = False
         self.__is_missing: bool = False
 
-    def date_to(self, dt: datetime):
+    def with_date_to(self, dt: datetime | None):
+        if dt is None:
+            self.__date_to = None
+            return
         self.__date_to = dt.astimezone(timezone.utc)
 
-    def date_from(self, dt: datetime):
+    def with_date_from(self, dt: datetime | None):
+        if dt is None:
+            self.__date_to = None
+            return
         self.__date_from = dt.astimezone(timezone.utc)
 
-    def is_deleted(self):
-        self.__is_deleted = True
+    def with_match_deleted(self, match_deleted: bool = True):
+        self.__is_deleted = match_deleted
+
+    def is_match_deleted(self) -> bool:
+        return self.__is_deleted
 
-    def is_missing(self):
-        self.__is_missing = True
+    def with_match_missing(self, match_missing: bool = True):
+        self.__is_missing = match_missing
+
+    def is_match_missing(self) -> bool:
+        return self.__is_missing
 
     def match(self, d: any) -> bool:
         d: dict[str, any]
+        """
+        :param d: data dict with keys: "link", "server-data", "message-id"
+        """
         link: LinkInterface = d["link"]
         if link.is_object():
             return True
@@ -43,15 +58,9 @@ def match(self, d: any) -> bool:
             ts2 = int(link.mutation()) / 1000.0
             if ts2 < ts1:
                 return False
-        if not self.__is_deleted and not self.__is_missing:
-            # no missing or deleted filter
-            return True
-        if self.__is_deleted and link.is_deleted():
-            # deleted
+        if link.is_deleted():
+            return self.is_match_deleted()
+        if not self.is_match_missing():
             return True
-        if self.__is_missing:
-            ids_from_server: dict[str, any] = d["server-data"]
-            if link.id() not in ids_from_server:
-                # missing
-                return True
-        return False
+        ids_from_server: dict[str, any] = d["server-data"]
+        return self.is_match_missing() and link.id() not in ids_from_server

gwbackupy/gmail.py

@@ -468,7 +468,8 @@ def __restore_message(
             logging.debug(f"{restore_message_id} {meta}")
             with self.storage.get(link[1]) as mf:
                 message_content = gzip.decompress(mf.read())
-            label_ids_from_message: [str] = meta["labelIds"]
+            # meta without labelIds is valid from server
+            label_ids_from_message: [str] = meta.get("labelIds", [])
             try:
                 label_ids_from_message.index("CHAT")
                 # not restorable as message
@@ -486,7 +487,7 @@ def __restore_message(
                 label_ids_from_message=label_ids_from_message,
             )
             label_names = []
-            for label_id in meta["labelIds"]:
+            for label_id in label_ids_from_message:
                 label_names.append(labels_form_server[label_id]["name"])
 
             logging.info(
@@ -540,18 +541,12 @@ def restore(
         self,
         item_filter: FilterInterface,
         to_email: str | None = None,
-        restore_deleted: bool = False,
         add_labels: list[str] | None = None,
-        restore_missing: bool = False,
     ):
         self.__error_count = 0
         if to_email is None:
             to_email = self.email
 
-        if not restore_deleted and not restore_missing:
-            logging.warning("Tasks not found, see more e.g. --restore-deleted")
-            return True
-
         logging.info("Scanning backup storage...")
         stored_data_all = self.storage.find()
         logging.info(f"Stored items: {len(stored_data_all)}")

gwbackupy/gwbackupy_cli.py

@@ -245,24 +245,30 @@ def cli_startup():
                     args.add_labels = ["gwbackupy"]
                 item_filter = GmailFilter()
                 if args.restore_deleted:
-                    item_filter.is_deleted()
+                    item_filter.with_match_deleted()
                     logging.info("Filter options: deleted")
                 if args.restore_missing:
-                    item_filter.is_missing()
+                    item_filter.with_match_missing()
                     logging.info("Filter options: missing")
                 if args.filter_date_from is not None:
                     dt = parse_date(args.filter_date_from, args.timezone)
-                    item_filter.date_from(dt)
+                    item_filter.with_date_from(dt)
                     logging.info(f"Filter options: date from {dt}")
                 if args.filter_date_to is not None:
                     dt = parse_date(args.filter_date_to, args.timezone)
-                    item_filter.date_to(dt)
+                    item_filter.with_date_to(dt)
                     logging.info(f"Filter options: date to {dt}")
+
+                if (
+                    not item_filter.is_match_deleted()
+                    and not item_filter.is_match_missing()
+                ):
+                    logging.warning("Tasks not found, see more e.g. --restore-deleted")
+                    return True
+
                 if gmail.restore(
                     to_email=args.to_email,
                     item_filter=item_filter,
-                    restore_deleted=args.restore_deleted,
-                    restore_missing=args.restore_missing,
                     add_labels=args.add_labels,
                 ):
                     exit(0)

gwbackupy/storage/file_storage.py

@@ -13,7 +13,6 @@
 
 from gwbackupy.storage.storage_interface import (
     StorageInterface,
-    Path,
     Data,
     LinkFilter,
     LinkInterface,

gwbackupy/tests/mock_storage.py

@@ -29,6 +29,12 @@ def fill(self, data: dict[str, any]):
     def mutation(self) -> str:
         return self.get_property(LinkInterface.property_mutation)
 
+    def set_mutation_timestamp(self, mutation: datetime):
+        if isinstance(mutation, datetime):
+            mutation = mutation.timestamp()
+        mutation = str(int(mutation * 1000))
+        self.set_properties({LinkInterface.property_mutation: mutation})
+
     def id(self) -> str:
         return self.__id
 

gwbackupy/tests/test_file_storage.py

@@ -132,6 +132,7 @@ def test_mutations():
         assert link2 in links
         deleted_found = False
         for link in links:
+            link: LinkInterface
             if link.is_deleted():
                 assert not deleted_found
                 deleted_found = True

gwbackupy/tests/test_gmail.py

@@ -146,18 +146,55 @@ def test_restore_with_label_recreate(to_email: str, clear_labels: bool):
         sw.inject_labels_clear()
     sw.inject_messages_clear()
     filtr = GmailFilter()
-    filtr.is_missing()
-    assert gmail.restore(filtr, restore_missing=True, add_labels=[], to_email=to_email)
+    filtr.with_match_missing()
+    assert gmail.restore(filtr, add_labels=[], to_email=to_email)
     messages = sw.get_messages(to_email, q="all")
     assert len(messages) == 1
     restored_message = list(messages.values())[0]
     restored_label_ids = restored_message["labelIds"]
     reloaded_labels = sw.get_labels(to_email)
-    assert len(restored_label_ids) >= 2
+    assert len(restored_label_ids) == 2
     assert __find_label_by_label_name(reloaded_labels, label1["name"])
     assert __find_label_by_label_name(reloaded_labels, label2["name"])
 
 
+def test_restore_without_label_ids_key():
+    """
+    Test that restore works when labelIds key is missing in message metadata
+    """
+    ms = MockStorage()
+    sw = MockGmailServiceWrapper()
+    message_id = random_string()
+    message_raw = bytes(f"Message body... {message_id}", "utf-8")
+    email = "example@example.com"
+    to_email = "example-to@example.com"
+    sw.inject_message(
+        email,
+        {
+            "id": message_id,
+            "raw": encode_base64url(message_raw),
+            "internalDate": str(int(datetime.now().timestamp() * 1000)),
+            "snippet": "A short snippet",
+        },
+    )
+    gmail = Gmail(
+        email=email,
+        storage=ms,
+        service_wrapper=sw,
+    )
+    assert gmail.backup()
+    sw.inject_messages_clear()
+    sw.inject_labels_clear()
+    filtr = GmailFilter()
+    filtr.with_match_missing()
+    assert gmail.restore(filtr, add_labels=[], to_email=to_email)
+    messages = sw.get_messages(to_email, q="all")
+    assert len(messages) == 1
+    restored_message = list(messages.values())[0]
+    restored_label_ids = restored_message["labelIds"]
+    assert len(restored_label_ids) == 0
+
+
 def __find_label_by_label_name(
     labels: List[Dict[str, any]], name: str
 ) -> Dict[str, any]: