From 53893f79e76509a60ae2df0bf2458d5fba533541 Mon Sep 17 00:00:00 2001 From: Alexander Kozlovsky Date: Wed, 10 Apr 2024 11:22:51 +0200 Subject: [PATCH 1/6] Cherry-pick: Add a sentry tag for the program file architecture (32-bit or 64-bit) --- src/tribler/core/sentry_reporter/sentry_reporter.py | 2 ++ src/tribler/gui/dialogs/feedbackdialog.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/tribler/core/sentry_reporter/sentry_reporter.py b/src/tribler/core/sentry_reporter/sentry_reporter.py index 995305ab3b4..80b0123456f 100644 --- a/src/tribler/core/sentry_reporter/sentry_reporter.py +++ b/src/tribler/core/sentry_reporter/sentry_reporter.py @@ -27,6 +27,7 @@ LAST_CORE_OUTPUT = 'last_core_output' LAST_PROCESSES = 'last_processes' PLATFORM = 'platform' +PROCESS_ARCHITECTURE = 'process_architecture' OS = 'os' MACHINE = 'machine' COMMENTS = 'comments' @@ -213,6 +214,7 @@ def send_event(self, event: Dict = None, post_data: Dict = None, sys_info: Dict # tags tags = event[TAGS] + tags[PROCESS_ARCHITECTURE] = get_value(post_data, PROCESS_ARCHITECTURE) tags[VERSION] = get_value(post_data, VERSION) tags[MACHINE] = get_value(post_data, MACHINE) tags[OS] = get_value(post_data, OS) diff --git a/src/tribler/gui/dialogs/feedbackdialog.py b/src/tribler/gui/dialogs/feedbackdialog.py index 4ba1d7e1008..1a6be0af412 100644 --- a/src/tribler/gui/dialogs/feedbackdialog.py +++ b/src/tribler/gui/dialogs/feedbackdialog.py @@ -11,7 +11,7 @@ from PyQt5.QtWidgets import QAction, QDialog, QMessageBox, QTreeWidgetItem from tribler.core.components.reporter.reported_error import ReportedError -from tribler.core.sentry_reporter.sentry_reporter import SentryReporter +from tribler.core.sentry_reporter.sentry_reporter import PROCESS_ARCHITECTURE, SentryReporter from tribler.core.sentry_reporter.sentry_scrubber import SentryScrubber from tribler.core.sentry_reporter.sentry_tools import CONTEXT_DELIMITER, LONG_TEXT_DELIMITER from tribler.gui.sentry_mixin import AddBreadcrumbOnShowMixin @@ -142,6 +142,7 @@ def on_send_clicked(self, checked): stack = self.error_text_edit.toPlainText() post_data = { + PROCESS_ARCHITECTURE: platform.architecture()[0], "version": self.tribler_version, "machine": platform.machine(), "os": platform.platform(), From 54939b7e1ba8025d207f772ca32b20e8fd382032 Mon Sep 17 00:00:00 2001 From: Alexander Kozlovsky Date: Fri, 12 Apr 2024 04:27:45 +0200 Subject: [PATCH 2/6] Cherry-pick: Reconfigure Sentry logs --- .../core/sentry_reporter/sentry_reporter.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/tribler/core/sentry_reporter/sentry_reporter.py b/src/tribler/core/sentry_reporter/sentry_reporter.py index 80b0123456f..bf502fba4df 100644 --- a/src/tribler/core/sentry_reporter/sentry_reporter.py +++ b/src/tribler/core/sentry_reporter/sentry_reporter.py @@ -9,6 +9,7 @@ from typing import Any, Dict, List, Optional import sentry_sdk +import sentry_sdk.utils from faker import Faker from sentry_sdk.integrations.logging import LoggingIntegration, ignore_logger from sentry_sdk.integrations.threading import ThreadingIntegration @@ -22,6 +23,25 @@ parse_stacktrace, ) + +def fix_sentry_logger(sentry_logger): + # Sentry log requires reconfiguration to be useful. By default, Sentry does not show even error-level log records + # until the debug option for Sentry is enabled. That means that unsuccessful attempts to send report are ignored. + # Enabling debug for Sentry is not recommended in production, and it sends the debug messages to stderr via + # a separate handler that is not in line with how Tribler handles logs. As a solution, Sentry developers recommend + # manual logger reconfiguration: https://github.com/getsentry/sentry-python/issues/1191#issuecomment-1023721841 + + for f in list(sentry_logger.filters): + sentry_logger.removeFilter(f) + + for h in list(sentry_logger.handlers): + sentry_logger.removeHandler(h) + + sentry_logger.setLevel(logging.WARNING) + + +fix_sentry_logger(sentry_sdk.utils.logger) + VALUE = 'value' TYPE = 'type' LAST_CORE_OUTPUT = 'last_core_output' From e41c634212680d086e9cb9e0f4cd8a59b1febe84 Mon Sep 17 00:00:00 2001 From: Alexander Kozlovsky Date: Fri, 12 Apr 2024 05:01:24 +0200 Subject: [PATCH 3/6] Cherry-pick: Fixes #7966: Fix Sentry report corruption by SentryScrubber --- .../core/sentry_reporter/sentry_scrubber.py | 9 ++++++-- .../core/sentry_reporter/sentry_tools.py | 3 --- .../tests/test_sentry_scrubber.py | 23 +++++++++++-------- .../tests/test_sentry_tools.py | 3 +-- 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/src/tribler/core/sentry_reporter/sentry_scrubber.py b/src/tribler/core/sentry_reporter/sentry_scrubber.py index a3efa7dd404..873adf9f404 100644 --- a/src/tribler/core/sentry_reporter/sentry_scrubber.py +++ b/src/tribler/core/sentry_reporter/sentry_scrubber.py @@ -172,17 +172,22 @@ def scrub_entity_recursively(self, entity: Union[str, Dict, List, Any], depth=10 if isinstance(entity, dict): result = {} for key, value in entity.items(): - if key in self.dict_keys_for_scrub: + if key in self.dict_keys_for_scrub and isinstance(value, str): value = value.strip() fake_value = obfuscate_string(value) placeholder = self.create_placeholder(fake_value) self.add_sensitive_pair(value, placeholder) - result[key] = self.scrub_entity_recursively(value, depth) + result[key] = placeholder + else: + result[key] = self.scrub_entity_recursively(value, depth) return result return entity def add_sensitive_pair(self, text, placeholder): + if not (text and text.strip()): # We should not replace empty substrings in the middle of other strings + return + if text in self.sensitive_occurrences: return diff --git a/src/tribler/core/sentry_reporter/sentry_tools.py b/src/tribler/core/sentry_reporter/sentry_tools.py index 1cbdf8ebe52..8c39ccf4e7d 100644 --- a/src/tribler/core/sentry_reporter/sentry_tools.py +++ b/src/tribler/core/sentry_reporter/sentry_tools.py @@ -200,9 +200,6 @@ def obfuscate_string(s: str, part_of_speech: str = 'noun') -> str: The same random words will be generated for the same given strings. """ - if not s: - return s - faker = Faker(locale='en_US') faker.seed_instance(s) return faker.word(part_of_speech=part_of_speech) diff --git a/src/tribler/core/sentry_reporter/tests/test_sentry_scrubber.py b/src/tribler/core/sentry_reporter/tests/test_sentry_scrubber.py index 6e37651a914..e1bd6f3752f 100644 --- a/src/tribler/core/sentry_reporter/tests/test_sentry_scrubber.py +++ b/src/tribler/core/sentry_reporter/tests/test_sentry_scrubber.py @@ -213,16 +213,16 @@ def test_scrub_event(scrubber): } assert scrubber.scrub_event(event) == { 'the very first item': '', - 'server_name': '', + 'server_name': '', CONTEXTS: { REPORTER: { 'any': { - 'USERNAME': '', + 'USERNAME': '', 'USERDOMAIN_ROAMINGPROFILE': '', 'PATH': '/users//apps', 'TMP_WIN': 'C:\\Users\\\\AppData\\Local\\Temp', - 'USERDOMAIN': '', - 'COMPUTERNAME': '', + 'USERDOMAIN': '', + 'COMPUTERNAME': '', }, STACKTRACE: [ 'Traceback (most recent call last):', @@ -301,15 +301,20 @@ def test_scrub_dict(scrubber): assert scrubber.scrub_entity_recursively(None) is None assert scrubber.scrub_entity_recursively({}) == {} - given = {'PATH': '/home/username/some/', 'USERDOMAIN': 'UD', 'USERNAME': 'U', 'REPEATED': 'user username UD U'} + assert scrubber.scrub_entity_recursively({'key': [1]}) == {'key': [1]} # non-string values should not lead to error + + given = {'PATH': '/home/username/some/', 'USERDOMAIN': 'UD', 'USERNAME': 'U', 'REPEATED': 'user username UD U', + 'key': ''} assert scrubber.scrub_entity_recursively(given) == {'PATH': '/home//some/', 'REPEATED': 'user ', 'USERDOMAIN': '', - 'USERNAME': ''} + 'USERNAME': '', + 'key': ''} - assert 'username' in scrubber.sensitive_occurrences.keys() - assert 'UD' in scrubber.sensitive_occurrences.keys() - assert 'U' in scrubber.sensitive_occurrences.keys() + assert 'username' in scrubber.sensitive_occurrences + assert 'UD' in scrubber.sensitive_occurrences + assert 'U' in scrubber.sensitive_occurrences + assert '' not in scrubber.sensitive_occurrences def test_scrub_list(scrubber): diff --git a/src/tribler/core/sentry_reporter/tests/test_sentry_tools.py b/src/tribler/core/sentry_reporter/tests/test_sentry_tools.py index 4812a157ddc..037821cadb7 100644 --- a/src/tribler/core/sentry_reporter/tests/test_sentry_tools.py +++ b/src/tribler/core/sentry_reporter/tests/test_sentry_tools.py @@ -144,8 +144,7 @@ def test_extract_dict(): OBFUSCATED_STRINGS = [ - (None, None), - ('', ''), + ('', 'dress'), ('any', 'challenge'), ('string', 'quality'), ] From b0d44965b9edfbe009c099abf7faeba69a7abe7f Mon Sep 17 00:00:00 2001 From: Alexander Kozlovsky Date: Wed, 10 Apr 2024 13:09:12 +0200 Subject: [PATCH 4/6] Cherry-pick: Show Tribler source lines in traceback when running Tribler from a bundle --- src/run_tribler.py | 2 ++ src/tribler/core/utilities/linecache_patch.py | 25 +++++++++++++ .../utilities/tests/test_linecache_patch.py | 36 +++++++++++++++++++ 3 files changed, 63 insertions(+) create mode 100644 src/tribler/core/utilities/linecache_patch.py create mode 100644 src/tribler/core/utilities/tests/test_linecache_patch.py diff --git a/src/run_tribler.py b/src/run_tribler.py index 51db913e779..4584ce0f6c1 100644 --- a/src/run_tribler.py +++ b/src/run_tribler.py @@ -11,6 +11,7 @@ from tribler.core.sentry_reporter.sentry_reporter import SentryReporter, SentryStrategy from tribler.core.sentry_reporter.sentry_scrubber import SentryScrubber +from tribler.core.utilities import linecache_patch from tribler.core.utilities.asyncio_fixes.finish_accept_patch import apply_finish_accept_patch from tribler.core.utilities.slow_coro_detection.main_thread_stack_tracking import start_main_thread_stack_tracing from tribler.core.utilities.osutils import get_root_state_directory @@ -78,6 +79,7 @@ def init_boot_logger(): def main(): + linecache_patch.patch() init_boot_logger() parsed_args = RunTriblerArgsParser().parse_args() diff --git a/src/tribler/core/utilities/linecache_patch.py b/src/tribler/core/utilities/linecache_patch.py new file mode 100644 index 00000000000..5645fc8c469 --- /dev/null +++ b/src/tribler/core/utilities/linecache_patch.py @@ -0,0 +1,25 @@ +import linecache +import sys + +original_updatecache = linecache.updatecache + + +def patched_updatecache(filename, *args, **kwargs): + if getattr(sys, 'frozen', False): + # When Tribler runs from a bundle, Tribler sources are available inside the `tribler_source` subfolder + if filename.startswith('src\\tribler'): # Relative path with cx_freeze on Windows + filename = 'tribler_source' + filename[3:] # Replacing `src\\` -> `tribler_source\\` + elif filename.startswith('tribler/'): # Relative path with PyInstaller on Mac/Linux: + filename = 'tribler_source/' + filename # Appending `tribler_source/` to the relative path + result = original_updatecache(filename, *args, **kwargs) + return result + + +patched_updatecache.patched = True + + +def patch(): + if getattr(linecache.updatecache, 'patched', False): + return + + linecache.updatecache = patched_updatecache diff --git a/src/tribler/core/utilities/tests/test_linecache_patch.py b/src/tribler/core/utilities/tests/test_linecache_patch.py new file mode 100644 index 00000000000..535df6f4e72 --- /dev/null +++ b/src/tribler/core/utilities/tests/test_linecache_patch.py @@ -0,0 +1,36 @@ +from unittest.mock import Mock, patch + +from tribler.core.utilities import linecache_patch + + +def original_updatecache_mock(filename, *args, **kwargs): + return [filename] # as if file consist of a single line that equal to the filename + + +@patch('tribler.core.utilities.linecache_patch.original_updatecache', original_updatecache_mock) +@patch('sys.frozen', False, create=True) +def test_not_frozen(): + assert linecache_patch.patched_updatecache('src\\tribler\\path') == ['src\\tribler\\path'] + assert linecache_patch.patched_updatecache('tribler/path') == ['tribler/path'] + assert linecache_patch.patched_updatecache('other/path') == ['other/path'] + + +@patch('tribler.core.utilities.linecache_patch.original_updatecache', original_updatecache_mock) +@patch('sys.frozen', True, create=True) +def test_frozen(): + assert linecache_patch.patched_updatecache('src\\tribler\\path') == ['tribler_source\\tribler\\path'] + assert linecache_patch.patched_updatecache('tribler/path') == ['tribler_source/tribler/path'] + assert linecache_patch.patched_updatecache('other/path') == ['other/path'] + + +@patch('tribler.core.utilities.linecache_patch.linecache') +def test_patch(linecache_mock): + _original_updatecache_mock = Mock(patched=True) + linecache_mock.updatecache = _original_updatecache_mock + + linecache_patch.patch() + assert linecache_mock.updatecache is _original_updatecache_mock # already patched, no second time patch + + _original_updatecache_mock.patched = False + linecache_patch.patch() + assert linecache_mock.updatecache is linecache_patch.patched_updatecache From 7c97859c965c9f4e919f8665874363e7a3501abe Mon Sep 17 00:00:00 2001 From: Alexander Kozlovsky Date: Wed, 17 Apr 2024 17:38:00 +0200 Subject: [PATCH 5/6] Cherry-pick: Fixes #7972: UDP server stops accepting datagrams from any clients after a single client disconnects (cherry picked from commit 449b864bac9bfa0a100deae7461423abd076e67d) --- src/run_tribler.py | 2 + .../asyncio_fixes/proactor_recvfrom_patch.py | 55 +++++++++++++ .../tests/test_proactor_recvfrom_patch.py | 79 +++++++++++++++++++ 3 files changed, 136 insertions(+) create mode 100644 src/tribler/core/utilities/asyncio_fixes/proactor_recvfrom_patch.py create mode 100644 src/tribler/core/utilities/asyncio_fixes/tests/test_proactor_recvfrom_patch.py diff --git a/src/run_tribler.py b/src/run_tribler.py index 4584ce0f6c1..a345f155c08 100644 --- a/src/run_tribler.py +++ b/src/run_tribler.py @@ -13,6 +13,7 @@ from tribler.core.sentry_reporter.sentry_scrubber import SentryScrubber from tribler.core.utilities import linecache_patch from tribler.core.utilities.asyncio_fixes.finish_accept_patch import apply_finish_accept_patch +from tribler.core.utilities.asyncio_fixes.proactor_recvfrom_patch import apply_proactor_recvfrom_patch from tribler.core.utilities.slow_coro_detection.main_thread_stack_tracking import start_main_thread_stack_tracing from tribler.core.utilities.osutils import get_root_state_directory from tribler.core.utilities.utilities import is_frozen @@ -97,6 +98,7 @@ def main(): if parsed_args.core: if sys.platform == 'win32': apply_finish_accept_patch() + apply_proactor_recvfrom_patch() from tribler.core.utilities.pony_utils import track_slow_db_sessions track_slow_db_sessions() diff --git a/src/tribler/core/utilities/asyncio_fixes/proactor_recvfrom_patch.py b/src/tribler/core/utilities/asyncio_fixes/proactor_recvfrom_patch.py new file mode 100644 index 00000000000..6d1121194f1 --- /dev/null +++ b/src/tribler/core/utilities/asyncio_fixes/proactor_recvfrom_patch.py @@ -0,0 +1,55 @@ +from asyncio.log import logger + +try: + import _overlapped +except ImportError: + _overlapped = None + + +NULL = 0 + +ERROR_PORT_UNREACHABLE = 1234 # _overlapped.ERROR_PORT_UNREACHABLE, available in Python >= 3.11 +ERROR_NETNAME_DELETED = 64 +ERROR_OPERATION_ABORTED = 995 + +patch_applied = False + + +def apply_proactor_recvfrom_patch(): # pragma: no cover + global patch_applied # pylint: disable=global-statement + if patch_applied: + return + + from asyncio import IocpProactor + + IocpProactor.recvfrom = patched_recvfrom + + patch_applied = True + logger.info("Patched IocpProactor.recvfrom to handle ERROR_PORT_UNREACHABLE") + + +def patched_recvfrom(self, conn, nbytes, flags=0): + self._register_with_iocp(conn) + ov = _overlapped.Overlapped(NULL) + try: + ov.WSARecvFrom(conn.fileno(), nbytes, flags) + except BrokenPipeError: + return self._result((b'', None)) + + def finish_recvfrom(trans, key, ov, error_class=OSError): + try: + return ov.getresult() + except error_class as exc: + if exc.winerror in (ERROR_NETNAME_DELETED, ERROR_OPERATION_ABORTED): + raise ConnectionResetError(*exc.args) + + # ******************** START OF THE PATCH ******************** + # WSARecvFrom will report ERROR_PORT_UNREACHABLE when the same + # socket was used to send to an address that is not listening. + if exc.winerror == ERROR_PORT_UNREACHABLE: + return b'', None # ignore the error + # ******************** END OF THE PATCH ********************** + + raise + + return self._register(ov, conn, finish_recvfrom) diff --git a/src/tribler/core/utilities/asyncio_fixes/tests/test_proactor_recvfrom_patch.py b/src/tribler/core/utilities/asyncio_fixes/tests/test_proactor_recvfrom_patch.py new file mode 100644 index 00000000000..7d5c2aab0f5 --- /dev/null +++ b/src/tribler/core/utilities/asyncio_fixes/tests/test_proactor_recvfrom_patch.py @@ -0,0 +1,79 @@ +from unittest.mock import Mock, patch + +import pytest + +from tribler.core.utilities.asyncio_fixes.proactor_recvfrom_patch import ERROR_NETNAME_DELETED, ERROR_OPERATION_ABORTED, \ + ERROR_PORT_UNREACHABLE, patched_recvfrom + + +@patch('tribler.core.utilities.asyncio_fixes.proactor_recvfrom_patch._overlapped') +def test_patched_recvfrom_broken_pipe_error(overlapped): + proactor, conn, nbytes, flags, ov = (Mock() for _ in range(5)) + overlapped.Overlapped.return_value = ov + conn.fileno.return_value = Mock() + ov.WSARecvFrom.side_effect = BrokenPipeError() + proactor._result.return_value = Mock() + + result = patched_recvfrom(proactor, conn, nbytes, flags) + + proactor._register_with_iocp.assert_called_with(conn) + overlapped.Overlapped.assert_called_with(0) + ov.WSARecvFrom.assert_called_with(conn.fileno.return_value, nbytes, flags) + proactor._result.assert_called_with((b'', None)) + assert result is proactor._result.return_value + + +@patch('tribler.core.utilities.asyncio_fixes.proactor_recvfrom_patch._overlapped') +def test_patched_recvfrom(overlapped): + proactor, conn, nbytes, flags, ov, trans, key = (Mock() for _ in range(7)) + overlapped.Overlapped.return_value = ov + conn.fileno.return_value = Mock() + proactor._register.return_value = Mock() + + result = patched_recvfrom(proactor, conn, nbytes, flags) + proactor._register.assert_called_once() + assert result is proactor._register.return_value + args = proactor._register.call_args.args + assert args[:2] == (ov, conn) and len(args) == 3 + + finish_recvfrom = args[2] + + class OSErrorMock(Exception): + def __init__(self, winerror): + self.winerror = winerror + + with patch('tribler.core.utilities.asyncio_fixes.proactor_recvfrom_patch.OSError', 'OSErrorMock'): + + # Should raise ConnectionResetError if ov.getresult() raises OSError with winerror=ERROR_NETNAME_DELETED + + ov.getresult.assert_not_called() + ov.getresult.side_effect = OSErrorMock(ERROR_NETNAME_DELETED) + with pytest.raises(ConnectionResetError): + finish_recvfrom(trans, key, ov, error_class=OSErrorMock) + + # Should raise ConnectionResetError if ov.getresult() raises OSError with winerror=ERROR_OPERATION_ABORTED + + ov.getresult.side_effect = OSErrorMock(ERROR_OPERATION_ABORTED) + with pytest.raises(ConnectionResetError): + finish_recvfrom(trans, key, ov, error_class=OSErrorMock) + + # Should return empty result if ov.getresult() raises OSError with winerror=ERROR_PORT_UNREACHABLE + + ov.getresult.side_effect = OSErrorMock(ERROR_PORT_UNREACHABLE) + result = finish_recvfrom(trans, key, ov, error_class=OSErrorMock) + assert result == (b'', None) + + # Should reraise any other OSError raised by ov.getresult() + + ov.getresult.side_effect = OSErrorMock(-1) + with pytest.raises(OSErrorMock): + finish_recvfrom(trans, key, ov, error_class=OSErrorMock) + + # Should return result of ov.getresult() if no exceptions arised + + ov.getresult.side_effect = None + ov.getresult.return_value = Mock() + result = finish_recvfrom(trans, key, ov) + assert result is ov.getresult.return_value + + assert ov.getresult.call_count == 5 From f4a7072161778cf3ac8434f4a98216f72b54dbb6 Mon Sep 17 00:00:00 2001 From: drew2a Date: Thu, 18 Apr 2024 15:45:50 +0200 Subject: [PATCH 6/6] Remove the torrent grouping based on names --- .../gui/widgets/tablecontentdelegate.py | 5 ----- src/tribler/gui/widgets/tablecontentmodel.py | 21 +++++-------------- 2 files changed, 5 insertions(+), 21 deletions(-) diff --git a/src/tribler/gui/widgets/tablecontentdelegate.py b/src/tribler/gui/widgets/tablecontentdelegate.py index 54455638fdd..891cd7854c1 100644 --- a/src/tribler/gui/widgets/tablecontentdelegate.py +++ b/src/tribler/gui/widgets/tablecontentdelegate.py @@ -379,11 +379,6 @@ def draw_title_and_tags(self, painter: QPainter, option: QStyleOptionViewItem, i debug = False # change to True to see the search rank of items and to highlight remote items item_name = data_item["name"] - group = data_item.get("group") - if group: - has_remote_items = any(group_item.get('remote') for group_item in group.values()) - item_name += f" (+ {len(group)} similar{' *' if debug and has_remote_items else ''})" - if debug: rank = data_item.get("rank") if rank is not None: diff --git a/src/tribler/gui/widgets/tablecontentmodel.py b/src/tribler/gui/widgets/tablecontentmodel.py index bc2cee5ce3b..ee618f95d0d 100644 --- a/src/tribler/gui/widgets/tablecontentmodel.py +++ b/src/tribler/gui/widgets/tablecontentmodel.py @@ -116,7 +116,6 @@ def __init__(self, parent=None): self.saved_scroll_state = None self.qt_object_destroyed = False - self.group_by_name = False self.sort_by_rank = False self.text_filter = '' @@ -204,7 +203,6 @@ def extract_unique_new_items(self, items: List, on_top: bool, remote: bool) -> T # Only add unique items to the table model and reverse mapping from unique ids to rows is built. insert_index = 0 if on_top else len(self.data_items) unique_new_items = [] - name_mapping = {item['name']: item for item in self.data_items} if self.group_by_name else {} now = time.time() for item in items: if remote: @@ -218,21 +216,12 @@ def extract_unique_new_items(self, items: List, on_top: bool, remote: bool) -> T item_uid = get_item_uid(item) if item_uid not in self.item_uid_map: - prev_item = name_mapping.get(item['name']) - if self.group_by_name and prev_item is not None and not on_top and prev_item['type'] == REGULAR_TORRENT: - group = prev_item.setdefault('group', {}) - if item_uid not in group: - group[item_uid] = item - else: - self.item_uid_map[item_uid] = insert_index - if 'infohash' in item: - self.item_uid_map[item['infohash']] = insert_index - unique_new_items.append(item) - - if self.group_by_name and item['type'] == REGULAR_TORRENT and prev_item is None: - name_mapping[item['name']] = item + self.item_uid_map[item_uid] = insert_index + if 'infohash' in item: + self.item_uid_map[item['infohash']] = insert_index + unique_new_items.append(item) - insert_index += 1 + insert_index += 1 return unique_new_items, insert_index def add_items(self, new_items, on_top=False, remote=False):