From b2e8656022c2dfb324360188fd57ba0facd1ab26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Somogyi?= Date: Wed, 10 May 2023 14:11:55 +0200 Subject: [PATCH 01/12] add test --- gwbackupy/tests/test_helpers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gwbackupy/tests/test_helpers.py b/gwbackupy/tests/test_helpers.py index 1d44979..5dd1b6a 100644 --- a/gwbackupy/tests/test_helpers.py +++ b/gwbackupy/tests/test_helpers.py @@ -89,6 +89,9 @@ def test_is_rate_limit_exceeded(): data2[0]["error"]["details"].append(dict()) e = HttpError(Resp(403, "Forbidden"), json.dumps(data2).encode("utf8")) assert not is_rate_limit_exceeded(e) + data2[0]["error"]["details"] = "not a list" + e = HttpError(Resp(403, "Forbidden"), json.dumps(data2).encode("utf8")) + assert not is_rate_limit_exceeded(e) def test_random_string(): From bb128f83402a0b26b0a8911ec9d505921e3d82c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Somogyi?= Date: Wed, 10 May 2023 18:57:29 +0200 Subject: [PATCH 02/12] add test --- gwbackupy/filters/gmail_filter.py | 22 ++++---- gwbackupy/tests/test_gmail_filter.py | 78 ++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 13 deletions(-) create mode 100644 gwbackupy/tests/test_gmail_filter.py diff --git a/gwbackupy/filters/gmail_filter.py b/gwbackupy/filters/gmail_filter.py index efc37cd..93f1d3b 100644 --- a/gwbackupy/filters/gmail_filter.py +++ b/gwbackupy/filters/gmail_filter.py @@ -27,8 +27,10 @@ def is_deleted(self): def is_missing(self): self.__is_missing = True - def match(self, d: any) -> bool: - d: dict[str, any] + def match(self, d: dict[str, any]) -> bool: + """ + :param d: data dict with keys: "link", "server-data", "message-id" + """ link: LinkInterface = d["link"] if link.is_object(): return True @@ -43,15 +45,9 @@ def match(self, d: any) -> bool: ts2 = int(link.mutation()) / 1000.0 if ts2 < ts1: return False - if not self.__is_deleted and not self.__is_missing: - # no missing or deleted filter + if link.is_deleted(): + return self.__is_deleted + if not self.__is_missing: return True - if self.__is_deleted and link.is_deleted(): - # deleted - return True - if self.__is_missing: - ids_from_server: dict[str, any] = d["server-data"] - if link.id() not in ids_from_server: - # missing - return True - return False + ids_from_server: dict[str, any] = d["server-data"] + return self.__is_missing and link.id() not in ids_from_server diff --git a/gwbackupy/tests/test_gmail_filter.py b/gwbackupy/tests/test_gmail_filter.py new file mode 100644 index 0000000..afdbda6 --- /dev/null +++ b/gwbackupy/tests/test_gmail_filter.py @@ -0,0 +1,78 @@ +from gwbackupy.filters.gmail_filter import GmailFilter +from gwbackupy.storage.storage_interface import LinkInterface +from gwbackupy.tests.mock_storage import MockStorage + + +def test_match_object(): + f = GmailFilter() + ms = MockStorage() + link = ms.new_link("abc", "json") + link.set_properties({LinkInterface.property_object: True}) + assert f.match({ + "message-id": "abc", + "link": link, + "server-data": {}, + }) + assert f.match({ + "message-id": "abc", + "link": link, + "server-data": {"abc": dict()}, + }) + + +def test_match_metadata(): + f = GmailFilter() + ms = MockStorage() + link = ms.new_link("abc", "json") + link.set_properties({LinkInterface.property_metadata: True}) + assert f.match({ + "message-id": "abc", + "link": link, + "server-data": {}, + }) + assert f.match({ + "message-id": "abc", + "link": link, + "server-data": {"abc": dict()}, + }) + + +def test_match_metadata_deleted(): + f = GmailFilter() + ms = MockStorage() + link = ms.new_link("abc", "json") + link.set_properties({ + LinkInterface.property_metadata: True, + LinkInterface.property_deleted: True, + }) + assert not f.match({ + "message-id": "abc", + "link": link, + "server-data": {}, + }) + f.is_deleted() + assert f.match({ + "message-id": "abc", + "link": link, + "server-data": {}, + }) + + +def test_match_metadata_missing(): + f = GmailFilter() + ms = MockStorage() + link = ms.new_link("abc", "json") + link.set_properties({ + LinkInterface.property_metadata: True, + }) + f.is_missing() + assert f.match({ + "message-id": "abc", + "link": link, + "server-data": {}, + }) + assert not f.match({ + "message-id": "abc", + "link": link, + "server-data": {"abc": dict()}, + }) From 8b22f26fca17ae0bf813cca19de427474ccd9ae6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Somogyi?= Date: Wed, 10 May 2023 18:59:34 +0200 Subject: [PATCH 03/12] code format --- gwbackupy/tests/test_gmail_filter.py | 114 ++++++++++++++++----------- 1 file changed, 67 insertions(+), 47 deletions(-) diff --git a/gwbackupy/tests/test_gmail_filter.py b/gwbackupy/tests/test_gmail_filter.py index afdbda6..47d0596 100644 --- a/gwbackupy/tests/test_gmail_filter.py +++ b/gwbackupy/tests/test_gmail_filter.py @@ -8,16 +8,20 @@ def test_match_object(): ms = MockStorage() link = ms.new_link("abc", "json") link.set_properties({LinkInterface.property_object: True}) - assert f.match({ - "message-id": "abc", - "link": link, - "server-data": {}, - }) - assert f.match({ - "message-id": "abc", - "link": link, - "server-data": {"abc": dict()}, - }) + assert f.match( + { + "message-id": "abc", + "link": link, + "server-data": {}, + } + ) + assert f.match( + { + "message-id": "abc", + "link": link, + "server-data": {"abc": dict()}, + } + ) def test_match_metadata(): @@ -25,54 +29,70 @@ def test_match_metadata(): ms = MockStorage() link = ms.new_link("abc", "json") link.set_properties({LinkInterface.property_metadata: True}) - assert f.match({ - "message-id": "abc", - "link": link, - "server-data": {}, - }) - assert f.match({ - "message-id": "abc", - "link": link, - "server-data": {"abc": dict()}, - }) + assert f.match( + { + "message-id": "abc", + "link": link, + "server-data": {}, + } + ) + assert f.match( + { + "message-id": "abc", + "link": link, + "server-data": {"abc": dict()}, + } + ) def test_match_metadata_deleted(): f = GmailFilter() ms = MockStorage() link = ms.new_link("abc", "json") - link.set_properties({ - LinkInterface.property_metadata: True, - LinkInterface.property_deleted: True, - }) - assert not f.match({ - "message-id": "abc", - "link": link, - "server-data": {}, - }) + link.set_properties( + { + LinkInterface.property_metadata: True, + LinkInterface.property_deleted: True, + } + ) + assert not f.match( + { + "message-id": "abc", + "link": link, + "server-data": {}, + } + ) f.is_deleted() - assert f.match({ - "message-id": "abc", - "link": link, - "server-data": {}, - }) + assert f.match( + { + "message-id": "abc", + "link": link, + "server-data": {}, + } + ) def test_match_metadata_missing(): f = GmailFilter() ms = MockStorage() link = ms.new_link("abc", "json") - link.set_properties({ - LinkInterface.property_metadata: True, - }) + link.set_properties( + { + LinkInterface.property_metadata: True, + } + ) f.is_missing() - assert f.match({ - "message-id": "abc", - "link": link, - "server-data": {}, - }) - assert not f.match({ - "message-id": "abc", - "link": link, - "server-data": {"abc": dict()}, - }) + assert f.match( + { + "message-id": "abc", + "link": link, + "server-data": {}, + } + ) + assert not f.match( + { + "message-id": "abc", + "link": link, + "server-data": {"abc": dict()}, + } + ) From d649f336d39ab7c02a1d7f7939d169c458d95d49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Somogyi?= Date: Thu, 11 May 2023 16:38:16 +0200 Subject: [PATCH 04/12] python 3.7 compatibility rollback --- gwbackupy/filters/gmail_filter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gwbackupy/filters/gmail_filter.py b/gwbackupy/filters/gmail_filter.py index 93f1d3b..7bc8662 100644 --- a/gwbackupy/filters/gmail_filter.py +++ b/gwbackupy/filters/gmail_filter.py @@ -27,7 +27,8 @@ def is_deleted(self): def is_missing(self): self.__is_missing = True - def match(self, d: dict[str, any]) -> bool: + def match(self, d: any) -> bool: + d: dict[str, any] """ :param d: data dict with keys: "link", "server-data", "message-id" """ From dba4344628fc467bcef03ac8b9102561c41c62f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Somogyi?= Date: Thu, 11 May 2023 16:47:18 +0200 Subject: [PATCH 05/12] refactor filter_interface.py --- gwbackupy/filters/filter_interface.py | 5 ++++- gwbackupy/filters/gmail_filter.py | 16 +++++++++++----- gwbackupy/gwbackupy_cli.py | 4 ++-- gwbackupy/tests/test_file_storage.py | 1 + gwbackupy/tests/test_gmail.py | 2 +- gwbackupy/tests/test_gmail_filter.py | 4 ++-- 6 files changed, 21 insertions(+), 11 deletions(-) diff --git a/gwbackupy/filters/filter_interface.py b/gwbackupy/filters/filter_interface.py index 8d70f34..988749c 100644 --- a/gwbackupy/filters/filter_interface.py +++ b/gwbackupy/filters/filter_interface.py @@ -14,7 +14,10 @@ def date_from(self, dt: datetime): def date_to(self, dt: datetime): pass - def is_deleted(self): + def with_match_deleted(self): + pass + + def is_match_deleted(self) -> bool: pass def match(self, d: any) -> bool: diff --git a/gwbackupy/filters/gmail_filter.py b/gwbackupy/filters/gmail_filter.py index 7bc8662..466e813 100644 --- a/gwbackupy/filters/gmail_filter.py +++ b/gwbackupy/filters/gmail_filter.py @@ -21,12 +21,18 @@ def date_to(self, dt: datetime): def date_from(self, dt: datetime): self.__date_from = dt.astimezone(timezone.utc) - def is_deleted(self): + def with_match_deleted(self): self.__is_deleted = True - def is_missing(self): + def is_match_deleted(self) -> bool: + return self.__is_deleted + + def with_match_missing(self): self.__is_missing = True + def is_match_missing(self) -> bool: + return self.__is_missing + def match(self, d: any) -> bool: d: dict[str, any] """ @@ -47,8 +53,8 @@ def match(self, d: any) -> bool: if ts2 < ts1: return False if link.is_deleted(): - return self.__is_deleted - if not self.__is_missing: + return self.is_match_deleted() + if not self.is_match_missing(): return True ids_from_server: dict[str, any] = d["server-data"] - return self.__is_missing and link.id() not in ids_from_server + return self.is_match_missing() and link.id() not in ids_from_server diff --git a/gwbackupy/gwbackupy_cli.py b/gwbackupy/gwbackupy_cli.py index e376128..b8a2184 100644 --- a/gwbackupy/gwbackupy_cli.py +++ b/gwbackupy/gwbackupy_cli.py @@ -245,10 +245,10 @@ def cli_startup(): args.add_labels = ["gwbackupy"] item_filter = GmailFilter() if args.restore_deleted: - item_filter.is_deleted() + item_filter.with_match_deleted() logging.info("Filter options: deleted") if args.restore_missing: - item_filter.is_missing() + item_filter.with_match_missing() logging.info("Filter options: missing") if args.filter_date_from is not None: dt = parse_date(args.filter_date_from, args.timezone) diff --git a/gwbackupy/tests/test_file_storage.py b/gwbackupy/tests/test_file_storage.py index 0624ed1..3c52270 100644 --- a/gwbackupy/tests/test_file_storage.py +++ b/gwbackupy/tests/test_file_storage.py @@ -132,6 +132,7 @@ def test_mutations(): assert link2 in links deleted_found = False for link in links: + link: LinkInterface if link.is_deleted(): assert not deleted_found deleted_found = True diff --git a/gwbackupy/tests/test_gmail.py b/gwbackupy/tests/test_gmail.py index 3b8eb34..b04d570 100644 --- a/gwbackupy/tests/test_gmail.py +++ b/gwbackupy/tests/test_gmail.py @@ -146,7 +146,7 @@ def test_restore_with_label_recreate(to_email: str, clear_labels: bool): sw.inject_labels_clear() sw.inject_messages_clear() filtr = GmailFilter() - filtr.is_missing() + filtr.with_match_missing() assert gmail.restore(filtr, restore_missing=True, add_labels=[], to_email=to_email) messages = sw.get_messages(to_email, q="all") assert len(messages) == 1 diff --git a/gwbackupy/tests/test_gmail_filter.py b/gwbackupy/tests/test_gmail_filter.py index 47d0596..0582cfa 100644 --- a/gwbackupy/tests/test_gmail_filter.py +++ b/gwbackupy/tests/test_gmail_filter.py @@ -62,7 +62,7 @@ def test_match_metadata_deleted(): "server-data": {}, } ) - f.is_deleted() + f.with_match_deleted() assert f.match( { "message-id": "abc", @@ -81,7 +81,7 @@ def test_match_metadata_missing(): LinkInterface.property_metadata: True, } ) - f.is_missing() + f.with_match_missing() assert f.match( { "message-id": "abc", From 73be58e8369a3ce8f827ceebbc24110b6951eb13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Somogyi?= Date: Thu, 11 May 2023 16:53:34 +0200 Subject: [PATCH 06/12] refactor filter_interface.py --- gwbackupy/filters/filter_interface.py | 6 ++++-- gwbackupy/filters/gmail_filter.py | 14 ++++++++++---- gwbackupy/gwbackupy_cli.py | 4 ++-- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/gwbackupy/filters/filter_interface.py b/gwbackupy/filters/filter_interface.py index 988749c..a666469 100644 --- a/gwbackupy/filters/filter_interface.py +++ b/gwbackupy/filters/filter_interface.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from datetime import datetime from builtins import bool @@ -8,10 +10,10 @@ class FilterInterface: def __init__(self): pass - def date_from(self, dt: datetime): + def with_date_from(self, dt: datetime | None): pass - def date_to(self, dt: datetime): + def with_date_to(self, dt: datetime | None): pass def with_match_deleted(self): diff --git a/gwbackupy/filters/gmail_filter.py b/gwbackupy/filters/gmail_filter.py index 466e813..acf6e7f 100644 --- a/gwbackupy/filters/gmail_filter.py +++ b/gwbackupy/filters/gmail_filter.py @@ -1,8 +1,8 @@ +from __future__ import annotations + from datetime import datetime, timezone from typing import Union -import tzlocal - from gwbackupy.filters.filter_interface import FilterInterface from gwbackupy.storage.storage_interface import LinkInterface @@ -15,10 +15,16 @@ def __init__(self): self.__is_deleted: bool = False self.__is_missing: bool = False - def date_to(self, dt: datetime): + def with_date_to(self, dt: datetime | None): + if dt is None: + self.__date_to = None + return self.__date_to = dt.astimezone(timezone.utc) - def date_from(self, dt: datetime): + def with_date_from(self, dt: datetime | None): + if dt is None: + self.__date_to = None + return self.__date_from = dt.astimezone(timezone.utc) def with_match_deleted(self): diff --git a/gwbackupy/gwbackupy_cli.py b/gwbackupy/gwbackupy_cli.py index b8a2184..1e8ef04 100644 --- a/gwbackupy/gwbackupy_cli.py +++ b/gwbackupy/gwbackupy_cli.py @@ -252,11 +252,11 @@ def cli_startup(): logging.info("Filter options: missing") if args.filter_date_from is not None: dt = parse_date(args.filter_date_from, args.timezone) - item_filter.date_from(dt) + item_filter.with_date_from(dt) logging.info(f"Filter options: date from {dt}") if args.filter_date_to is not None: dt = parse_date(args.filter_date_to, args.timezone) - item_filter.date_to(dt) + item_filter.with_date_to(dt) logging.info(f"Filter options: date to {dt}") if gmail.restore( to_email=args.to_email, From b3d2c9aaa188e316cf3854783e14c190405becad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Somogyi?= Date: Thu, 11 May 2023 16:57:37 +0200 Subject: [PATCH 07/12] enh: extend FilterInterface and GmailFilter --- gwbackupy/filters/filter_interface.py | 2 +- gwbackupy/filters/gmail_filter.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/gwbackupy/filters/filter_interface.py b/gwbackupy/filters/filter_interface.py index a666469..f7b45d7 100644 --- a/gwbackupy/filters/filter_interface.py +++ b/gwbackupy/filters/filter_interface.py @@ -16,7 +16,7 @@ def with_date_from(self, dt: datetime | None): def with_date_to(self, dt: datetime | None): pass - def with_match_deleted(self): + def with_match_deleted(self, match_deleted: bool = True): pass def is_match_deleted(self) -> bool: diff --git a/gwbackupy/filters/gmail_filter.py b/gwbackupy/filters/gmail_filter.py index acf6e7f..cbb89c0 100644 --- a/gwbackupy/filters/gmail_filter.py +++ b/gwbackupy/filters/gmail_filter.py @@ -27,14 +27,14 @@ def with_date_from(self, dt: datetime | None): return self.__date_from = dt.astimezone(timezone.utc) - def with_match_deleted(self): - self.__is_deleted = True + def with_match_deleted(self, match_deleted: bool = True): + self.__is_deleted = match_deleted def is_match_deleted(self) -> bool: return self.__is_deleted - def with_match_missing(self): - self.__is_missing = True + def with_match_missing(self, match_missing: bool = True): + self.__is_missing = match_missing def is_match_missing(self) -> bool: return self.__is_missing From 8e148a53b843c8c09adb37c147e40f0688561403 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Somogyi?= Date: Thu, 11 May 2023 16:57:55 +0200 Subject: [PATCH 08/12] test: add more tests --- gwbackupy/tests/test_gmail_filter.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/gwbackupy/tests/test_gmail_filter.py b/gwbackupy/tests/test_gmail_filter.py index 0582cfa..576a99f 100644 --- a/gwbackupy/tests/test_gmail_filter.py +++ b/gwbackupy/tests/test_gmail_filter.py @@ -96,3 +96,17 @@ def test_match_metadata_missing(): "server-data": {"abc": dict()}, } ) + + +def test_with_methods(): + f = GmailFilter() + assert not f.is_match_deleted() + f.with_match_deleted() + assert f.is_match_deleted() + f.with_match_deleted(False) + assert not f.is_match_deleted() + assert not f.is_match_missing() + f.with_match_missing() + assert f.is_match_missing() + f.with_match_missing(False) + assert not f.is_match_missing() From 5f232868d79895db50be88ea5bb5330ceb17b54d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Somogyi?= Date: Thu, 11 May 2023 17:00:02 +0200 Subject: [PATCH 09/12] code: optimize imports --- gwbackupy/storage/file_storage.py | 1 - 1 file changed, 1 deletion(-) diff --git a/gwbackupy/storage/file_storage.py b/gwbackupy/storage/file_storage.py index 6846e65..fa3b514 100644 --- a/gwbackupy/storage/file_storage.py +++ b/gwbackupy/storage/file_storage.py @@ -13,7 +13,6 @@ from gwbackupy.storage.storage_interface import ( StorageInterface, - Path, Data, LinkFilter, LinkInterface, From c7c42f54aa3b9eb520873798b7fe70fb33d87c27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Somogyi?= Date: Fri, 12 May 2023 04:51:47 +0200 Subject: [PATCH 10/12] enh: remove gmail restore missing and delete method parameters #74 --- gwbackupy/gmail.py | 6 ------ gwbackupy/gwbackupy_cli.py | 10 ++++++++-- gwbackupy/tests/test_gmail.py | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/gwbackupy/gmail.py b/gwbackupy/gmail.py index d739863..d56170f 100644 --- a/gwbackupy/gmail.py +++ b/gwbackupy/gmail.py @@ -540,18 +540,12 @@ def restore( self, item_filter: FilterInterface, to_email: str | None = None, - restore_deleted: bool = False, add_labels: list[str] | None = None, - restore_missing: bool = False, ): self.__error_count = 0 if to_email is None: to_email = self.email - if not restore_deleted and not restore_missing: - logging.warning("Tasks not found, see more e.g. --restore-deleted") - return True - logging.info("Scanning backup storage...") stored_data_all = self.storage.find() logging.info(f"Stored items: {len(stored_data_all)}") diff --git a/gwbackupy/gwbackupy_cli.py b/gwbackupy/gwbackupy_cli.py index 1e8ef04..eebb459 100644 --- a/gwbackupy/gwbackupy_cli.py +++ b/gwbackupy/gwbackupy_cli.py @@ -258,11 +258,17 @@ def cli_startup(): dt = parse_date(args.filter_date_to, args.timezone) item_filter.with_date_to(dt) logging.info(f"Filter options: date to {dt}") + + if ( + not item_filter.is_match_deleted() + and not item_filter.is_match_missing() + ): + logging.warning("Tasks not found, see more e.g. --restore-deleted") + return True + if gmail.restore( to_email=args.to_email, item_filter=item_filter, - restore_deleted=args.restore_deleted, - restore_missing=args.restore_missing, add_labels=args.add_labels, ): exit(0) diff --git a/gwbackupy/tests/test_gmail.py b/gwbackupy/tests/test_gmail.py index b04d570..4ac4df3 100644 --- a/gwbackupy/tests/test_gmail.py +++ b/gwbackupy/tests/test_gmail.py @@ -147,7 +147,7 @@ def test_restore_with_label_recreate(to_email: str, clear_labels: bool): sw.inject_messages_clear() filtr = GmailFilter() filtr.with_match_missing() - assert gmail.restore(filtr, restore_missing=True, add_labels=[], to_email=to_email) + assert gmail.restore(filtr, add_labels=[], to_email=to_email) messages = sw.get_messages(to_email, q="all") assert len(messages) == 1 restored_message = list(messages.values())[0] From 75aac7251c6ee37b5b6abc6cb5fb02e4739f8005 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Somogyi?= Date: Fri, 12 May 2023 05:19:17 +0200 Subject: [PATCH 11/12] test: add more gmail filter tests --- gwbackupy/tests/mock_storage.py | 6 +++ gwbackupy/tests/test_gmail_filter.py | 78 ++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/gwbackupy/tests/mock_storage.py b/gwbackupy/tests/mock_storage.py index 096172e..58d18b9 100644 --- a/gwbackupy/tests/mock_storage.py +++ b/gwbackupy/tests/mock_storage.py @@ -29,6 +29,12 @@ def fill(self, data: dict[str, any]): def mutation(self) -> str: return self.get_property(LinkInterface.property_mutation) + def set_mutation_timestamp(self, mutation: datetime): + if isinstance(mutation, datetime): + mutation = mutation.timestamp() + mutation = str(int(mutation * 1000)) + self.set_properties({LinkInterface.property_mutation: mutation}) + def id(self) -> str: return self.__id diff --git a/gwbackupy/tests/test_gmail_filter.py b/gwbackupy/tests/test_gmail_filter.py index 576a99f..25c4d61 100644 --- a/gwbackupy/tests/test_gmail_filter.py +++ b/gwbackupy/tests/test_gmail_filter.py @@ -1,3 +1,5 @@ +from datetime import datetime, timezone + from gwbackupy.filters.gmail_filter import GmailFilter from gwbackupy.storage.storage_interface import LinkInterface from gwbackupy.tests.mock_storage import MockStorage @@ -110,3 +112,79 @@ def test_with_methods(): assert f.is_match_missing() f.with_match_missing(False) assert not f.is_match_missing() + + +def test_with_date_to(): + f = GmailFilter() + ms = MockStorage() + link = ms.new_link("abc", "json") + filter_date = datetime(2020, 1, 1, tzinfo=timezone.utc) + if datetime.now().timestamp() < filter_date.timestamp(): + raise Exception("The test is not valid if the filter date is in the future") + f.with_date_to(filter_date) + assert not f.match( + { + "message-id": "abc", + "link": link, + "server-data": {}, + } + ) + f.with_date_to(None) + assert f.match( + { + "message-id": "abc", + "link": link, + "server-data": {}, + } + ) + + +def test_with_date_from(): + f = GmailFilter() + ms = MockStorage() + f.with_date_from(datetime(2021, 1, 1, tzinfo=timezone.utc)) + link = ms.new_link( + "abc", + "json", + created_timestamp=datetime(2020, 1, 1, tzinfo=timezone.utc).timestamp(), + ) + link.set_mutation_timestamp(datetime(2019, 1, 1, tzinfo=timezone.utc)) + assert not f.match( + { + "message-id": "abc", + "link": link, + "server-data": {}, + } + ) + f.with_date_from(None) + assert f.match( + { + "message-id": "abc", + "link": ms.new_link("abc", "json"), + "server-data": {}, + } + ) + + +def test_with_date_from_and_to(): + f = GmailFilter() + ms = MockStorage() + link = ms.new_link("abc", "json") + link.set_mutation_timestamp(datetime(2020, 1, 1, tzinfo=timezone.utc)) + f.with_date_from(datetime(2021, 1, 1, tzinfo=timezone.utc)) + f.with_date_to(datetime(2022, 2, 1, tzinfo=timezone.utc)) + assert not f.match( + { + "message-id": "abc", + "link": link, + "server-data": {}, + } + ) + link.set_mutation_timestamp(datetime(2021, 5, 5, tzinfo=timezone.utc)) + assert f.match( + { + "message-id": "abc", + "link": link, + "server-data": {}, + } + ) From c3ea89a54a36c792583ba1216702d441853da1a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Somogyi?= Date: Fri, 12 May 2023 05:44:43 +0200 Subject: [PATCH 12/12] bug: fix gmail restore without labelIds in metadata (valid from server) --- CHANGELOG.md | 1 + gwbackupy/gmail.py | 5 +++-- gwbackupy/tests/test_gmail.py | 39 ++++++++++++++++++++++++++++++++++- 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 44afeac..e43264a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## DEV +- Bug: fix gmail restore without labelIds in stored message metadata ## 0.11.0 diff --git a/gwbackupy/gmail.py b/gwbackupy/gmail.py index d56170f..4a9fbb3 100644 --- a/gwbackupy/gmail.py +++ b/gwbackupy/gmail.py @@ -468,7 +468,8 @@ def __restore_message( logging.debug(f"{restore_message_id} {meta}") with self.storage.get(link[1]) as mf: message_content = gzip.decompress(mf.read()) - label_ids_from_message: [str] = meta["labelIds"] + # meta without labelIds is valid from server + label_ids_from_message: [str] = meta.get("labelIds", []) try: label_ids_from_message.index("CHAT") # not restorable as message @@ -486,7 +487,7 @@ def __restore_message( label_ids_from_message=label_ids_from_message, ) label_names = [] - for label_id in meta["labelIds"]: + for label_id in label_ids_from_message: label_names.append(labels_form_server[label_id]["name"]) logging.info( diff --git a/gwbackupy/tests/test_gmail.py b/gwbackupy/tests/test_gmail.py index 4ac4df3..59475a0 100644 --- a/gwbackupy/tests/test_gmail.py +++ b/gwbackupy/tests/test_gmail.py @@ -153,11 +153,48 @@ def test_restore_with_label_recreate(to_email: str, clear_labels: bool): restored_message = list(messages.values())[0] restored_label_ids = restored_message["labelIds"] reloaded_labels = sw.get_labels(to_email) - assert len(restored_label_ids) >= 2 + assert len(restored_label_ids) == 2 assert __find_label_by_label_name(reloaded_labels, label1["name"]) assert __find_label_by_label_name(reloaded_labels, label2["name"]) +def test_restore_without_label_ids_key(): + """ + Test that restore works when labelIds key is missing in message metadata + """ + ms = MockStorage() + sw = MockGmailServiceWrapper() + message_id = random_string() + message_raw = bytes(f"Message body... {message_id}", "utf-8") + email = "example@example.com" + to_email = "example-to@example.com" + sw.inject_message( + email, + { + "id": message_id, + "raw": encode_base64url(message_raw), + "internalDate": str(int(datetime.now().timestamp() * 1000)), + "snippet": "A short snippet", + }, + ) + gmail = Gmail( + email=email, + storage=ms, + service_wrapper=sw, + ) + assert gmail.backup() + sw.inject_messages_clear() + sw.inject_labels_clear() + filtr = GmailFilter() + filtr.with_match_missing() + assert gmail.restore(filtr, add_labels=[], to_email=to_email) + messages = sw.get_messages(to_email, q="all") + assert len(messages) == 1 + restored_message = list(messages.values())[0] + restored_label_ids = restored_message["labelIds"] + assert len(restored_label_ids) == 0 + + def __find_label_by_label_name( labels: List[Dict[str, any]], name: str ) -> Dict[str, any]: