Merge branch 'release/7.13' into main

README.rst

@@ -7,7 +7,7 @@ Tribler
 
 |downloads_7_0| |downloads_7_1| |downloads_7_2| |downloads_7_3| |downloads_7_4|
 |downloads_7_5| |downloads_7_6| |downloads_7_7| |downloads_7_8| |downloads_7_9|
-|downloads_7_10| |downloads_7_11| |downloads_7_12|
+|downloads_7_10| |downloads_7_11| |downloads_7_12| |downloads_7_13|
 
 |doi| |openhub| |discord|
 
@@ -187,6 +187,10 @@ We like to hear your feedback and suggestions. To reach out to us, you can join
  :target: https://github.com/Tribler/tribler/releases
  :alt: Downloads(7.12.1)
 
+.. |downloads_7_13| image:: https://img.shields.io/github/downloads/tribler/tribler/v7.13.0/total.svg?style=flat
+ :target: https://github.com/Tribler/tribler/releases
+ :alt: Downloads(7.13.0)
+
 .. |contributors| image:: https://img.shields.io/github/contributors/tribler/tribler.svg?style=flat
  :target: https://github.com/Tribler/tribler/graphs/contributors
  :alt: Contributors

build/mac/makedist_macos.sh

@@ -21,7 +21,7 @@ export RESOURCES=build/mac/resources
 
 python3 -m venv build-env
 . ./build-env/bin/activate
-python3 -m pip install --upgrade pip
+python3 -m pip install pip==23.0.1 # pin pip version to avoid "--no-use-pep517" issues with the latest version
 python3 -m pip install PyInstaller==4.2 --no-use-pep517
 python3 -m pip install --upgrade -r requirements-build.txt
 

doc/development/development_on_osx.rst

@@ -19,6 +19,12 @@ If you wish to run the Tribler Graphical User Interface, PyQt5 should be availab
  qmake --version # test whether qt is installed correctly
 
 
+Add `qt@5/bin` to the PATH environment variable, e.g.:
+
+.. code-block:: bash
+
+ export PATH="/usr/local/opt/qt@5/bin:$PATH"
+
 Other Packages
 ~~~~~~~~~~~~~~
 

src/tribler/core/components/component.py

@@ -2,48 +2,24 @@
 
 import logging
 import sys
+import time
 from asyncio import Event
 from typing import Optional, Set, TYPE_CHECKING, Type, Union
 
+from tribler.core.components.exceptions import ComponentStartupException, MissedDependency, NoneComponent
+from tribler.core.components.reporter.exception_handler import default_core_exception_handler
+from tribler.core.sentry_reporter.sentry_reporter import SentryReporter
+
 if TYPE_CHECKING:
  from tribler.core.components.session import Session, T
 
 
-class ComponentError(Exception):
- pass
-
-
-class ComponentStartupException(ComponentError):
- def __init__(self, component: Component, cause: Exception):
- super().__init__(component.__class__.__name__)
- self.component = component
- self.__cause__ = cause
-
-
-class MissedDependency(ComponentError):
- def __init__(self, component: Component, dependency: Type[Component]):
- msg = f'Missed dependency: {component.__class__.__name__} requires {dependency.__name__} to be active'
- super().__init__(msg)
- self.component = component
- self.dependency = dependency
-
-
-class MultipleComponentsFound(ComponentError):
- def __init__(self, comp_cls: Type[Component], candidates: Set[Component]):
- msg = f'Found multiple subclasses for the class {comp_cls}. Candidates are: {candidates}.'
- super().__init__(msg)
-
-
-class NoneComponent:
- def __getattr__(self, item):
- return NoneComponent()
-
-
 class Component:
  tribler_should_stop_on_component_error = True
 
- def __init__(self):
- self.logger = logging.getLogger(self.__class__.__name__)
+ def __init__(self, reporter: Optional[SentryReporter] = None):
+ self.name = self.__class__.__name__
+ self.logger = logging.getLogger(self.name)
  self.logger.info('__init__')
  self.session: Optional[Session] = None
  self.dependencies: Set[Component] = set()
@@ -54,20 +30,21 @@ def __init__(self):
  self.stopped = False
  # Every component starts unused, so it does not lock the whole system on shutdown
  self.unused_event.set()
+ self.reporter = reporter or default_core_exception_handler.sentry_reporter
 
  async def start(self):
- self.logger.info(f'Start: {self.__class__.__name__}')
+ start_time = time.time()
+ self._set_component_status('starting...')
  try:
  await self.run()
+ self._set_component_status(f'started in {time.time() - start_time:.4f}s')
  except Exception as e: # pylint: disable=broad-except
  # Writing to stderr is for the case when logger is not configured properly (as my happen in local tests,
  # for example) to avoid silent suppression of the important exceptions
- sys.stderr.write(f'\nException in {self.__class__.__name__}.start(): {type(e).__name__}:{e}\n')
- if isinstance(e, MissedDependency):
- # Use logger.error instead of logger.exception here to not spam log with multiple error tracebacks
- self.logger.error(e)
- else:
- self.logger.exception(f'Exception in {self.__class__.__name__}.start(): {type(e).__name__}:{e}')
+ sys.stderr.write(f'\nException in {self.name}.start(): {type(e).__name__}:{e}\n')
+ msg = f'exception in {self.name}.start(): {type(e).__name__}:{e}'
+ exc_info = not isinstance(e, MissedDependency)
+ self._set_component_status(msg, logging.ERROR, exc_info=exc_info)
  self.failed = True
  self.started_event.set()
  if self.session.failfast:
@@ -76,23 +53,25 @@ async def start(self):
  self.started_event.set()
 
  async def stop(self):
- component_name = self.__class__.__name__
  dependants = sorted(component.__class__.__name__ for component in self.reverse_dependencies)
- self.logger.info(f'Stopping {component_name}: waiting for {dependants} to release it')
+ msg = f'Stopping {self.name}: waiting for {dependants} to release it'
+ self._set_component_status(msg)
  await self.unused_event.wait()
- self.logger.info(f"Component {component_name} free, shutting down")
+ self._set_component_status('shutting down')
  try:
  await self.shutdown()
+ self._set_component_status('shut down')
  except Exception as e: # pylint: disable=broad-except
- self.logger.exception(f"Exception in {self.__class__.__name__}.shutdown(): {type(e).__name__}:{e}")
+ msg = f"exception in {self.name}.shutdown(): {type(e).__name__}:{e}"
+ self._set_component_status(msg, logging.ERROR, exc_info=True)
  raise
  finally:
  self.stopped = True
  for dep in list(self.dependencies):
  self._release_instance(dep)
  remaining_components = sorted(
  c.__class__.__name__ for c in self.session.components.values() if not c.stopped)
- self.logger.info(f"Component {component_name}, stopped. Remaining components: {remaining_components}")
+ self.logger.info(f"Component {self.name}, stopped. Remaining components: {remaining_components}")
 
  async def run(self):
  pass
@@ -123,9 +102,11 @@ async def get_component(self, dependency: Type[T]) -> Optional[T]:
  if not dep:
  return None
 
+ self._set_component_status(f'waiting for {dep.name}')
  await dep.started_event.wait()
+
  if dep.failed:
- self.logger.warning(f'Component {self.__class__.__name__} has failed dependency {dependency.__name__}')
+ self.logger.warning(f'Component {self.name} has failed dependency {dependency.__name__}')
  return None
 
  if dep not in self.dependencies and dep is not self:
@@ -166,3 +147,7 @@ def _unuse_by(self, component: Component):
  self.reverse_dependencies.remove(component)
  if not self.reverse_dependencies:
  self.unused_event.set()
+
+ def _set_component_status(self, status: str, log_level: int = logging.INFO, **kwargs):
+ self.reporter.additional_information['components_status'][self.name] = status
+ self.logger.log(log_level, f'{self.name}: {status}', **kwargs)

src/tribler/core/components/exceptions.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+from typing import Set, TYPE_CHECKING, Type
+
+if TYPE_CHECKING:
+ from tribler.core.components.component import Component
+
+
+class ComponentError(Exception):
+ pass
+
+
+class ComponentStartupException(ComponentError):
+ def __init__(self, component: Component, cause: Exception):
+ super().__init__(component.__class__.__name__)
+ self.component = component
+ self.__cause__ = cause
+
+
+class MissedDependency(ComponentError):
+ def __init__(self, component: Component, dependency: Type[Component]):
+ msg = f'Missed dependency: {component.__class__.__name__} requires {dependency.__name__} to be active'
+ super().__init__(msg)
+ self.component = component
+ self.dependency = dependency
+
+
+class MultipleComponentsFound(ComponentError):
+ def __init__(self, comp_cls: Type[Component], candidates: Set[Component]):
+ msg = f'Found multiple subclasses for the class {comp_cls}. Candidates are: {candidates}.'
+ super().__init__(msg)
+
+
+class NoneComponent:
+ def __getattr__(self, item):
+ return NoneComponent()

src/tribler/core/components/gigachannel_manager/gigachannel_manager.py

@@ -191,7 +191,6 @@ def check_channels_updates(self):
  infohash = bytes(channel.infohash)
  if self.download_manager.metainfo_requests.get(infohash):
  continue
- status = self.download_manager.get_download(infohash).get_state().get_status()
  if not self.download_manager.download_exists(infohash):
  self._logger.info(
  "Downloading new channel version %s ver %i->%i",
@@ -200,18 +199,20 @@ def check_channels_updates(self):
  channel.timestamp,
  )
  self.download_channel(channel)
- elif status == DownloadStatus.SEEDING:
+ continue
+
+ channel_download = self.download_manager.get_download(infohash)
+ if channel_download and channel_download.get_state().get_status() == DownloadStatus.SEEDING:
  self._logger.info(
  "Processing previously downloaded, but unprocessed channel torrent %s ver %i->%i",
  channel.dirname,
  channel.local_version,
  channel.timestamp,
  )
  self.channels_processing_queue[channel.infohash] = (PROCESS_CHANNEL_DIR, channel)
- except Exception:
- self._logger.exception(
- "Error when tried to download a newer version of channel %s", hexlify(channel.public_key)
- )
+ except Exception as e:
+ self._logger.exception("Error when tried to download a newer version of channel "
+ f"{hexlify(channel.public_key)}: {type(e).__name__}: {e}")
 
  async def remove_channel_download(self, to_remove):
  """

src/tribler/core/components/gigachannel_manager/tests/test_gigachannel_manager.py

@@ -1,4 +1,6 @@
 import asyncio
+import os
+import random
 from asyncio import Future
 from datetime import datetime
 from pathlib import Path
@@ -38,8 +40,8 @@ def personal_channel(metadata_store):
  return chan
 
 
-@pytest.fixture
-async def gigachannel_manager(metadata_store):
+@pytest.fixture(name="gigachannel_manager")
+async def gigachannel_manager_fixture(metadata_store):
  chanman = GigaChannelManager(
  state_dir=metadata_store.channels_dir.parent,
  metadata_store=metadata_store,
@@ -225,6 +227,60 @@ def mock_process_channel_dir(c, _):
  assert not gigachannel_manager.channels_processing_queue
 
 
+@db_session
+def test_check_channel_updates_for_different_states(gigachannel_manager, metadata_store):
+ def random_subscribed_channel():
+ return metadata_store.ChannelMetadata(
+ title=f"Channel {random.randint(0, 100)}",
+ public_key=os.urandom(32),
+ signature=os.urandom(32),
+ skip_key_check=True,
+ timestamp=123,
+ local_version=122,
+ subscribed=True,
+ infohash=random_infohash(),
+ )
+
+ # Three channels in different states based on the setup
+ channel_with_metainfo = random_subscribed_channel()
+ already_downloaded_channel = random_subscribed_channel()
+ non_downloaded_channel = random_subscribed_channel()
+
+ # Setup 1: metainfo is already available for channel torrent.
+ def mock_get_metainfo(infohash):
+ return MagicMock() if infohash == channel_with_metainfo.infohash else None
+
+ gigachannel_manager.download_manager.metainfo_requests = MagicMock(get=mock_get_metainfo)
+
+ # Setup 2: Only one specific channel torrent is already downloaded.
+ def mock_download_exists(infohash):
+ return infohash == already_downloaded_channel.infohash
+
+ gigachannel_manager.download_manager.download_exists = mock_download_exists
+
+ # Setup 2 (contd): We expect non-downloaded channel to be downloaded
+ # so mocking download_channel() method.
+ gigachannel_manager.download_channel = MagicMock()
+
+ # Setup 3: Downloaded channel torrent is set on Seeding state.
+ def mock_get_download(infohash):
+ if infohash != already_downloaded_channel.infohash:
+ return None
+
+ seeding_state = MagicMock(get_status=lambda: DownloadStatus.SEEDING)
+ return MagicMock(get_state=lambda: seeding_state)
+
+ gigachannel_manager.download_manager.get_download = mock_get_download
+
+ # Act
+ gigachannel_manager.check_channels_updates()
+
+ # Assert
+ gigachannel_manager.download_channel.assert_called_once_with(non_downloaded_channel)
+ assert len(gigachannel_manager.channels_processing_queue) == 1
+ assert already_downloaded_channel.infohash in gigachannel_manager.channels_processing_queue
+
+
 async def test_remove_cruft_channels(torrent_template, personal_channel, gigachannel_manager, metadata_store):
  remove_list = []
  with db_session:

src/tribler/core/components/knowledge/db/knowledge_db.py

@@ -2,7 +2,7 @@
 import logging
 from dataclasses import dataclass
 from enum import IntEnum
-from typing import Callable, Iterator, List, Optional, Set
+from typing import Any, Callable, Iterator, List, Optional, Set
 
 from pony import orm
 from pony.orm import raw_sql
@@ -139,6 +139,10 @@ class StatementOp(db.Entity):
 
  orm.composite_key(statement, peer)
 
+ class Misc(db.Entity): # pylint: disable=unused-variable
+ name = orm.PrimaryKey(str)
+ value = orm.Optional(str)
+
  def add_operation(self, operation: StatementOperation, signature: bytes, is_local_peer: bool = False,
  is_auto_generated: bool = False, counter_increment: int = 1) -> bool:
  """ Add the operation that will be applied to a statement.
@@ -441,3 +445,11 @@ def _get_random_operations_by_condition(self, condition: Callable[[Entity], bool
  operations.add(operation)
 
  return operations
+
+ def get_misc(self, key: str, default: Optional[str] = None) -> Optional[str]:
+ data = self.instance.Misc.get(name=key)
+ return data.value if data else default
+
+ def set_misc(self, key: str, value: Any):
+ key_value = get_or_create(self.instance.Misc, name=key)
+ key_value.value = str(value)

src/tribler/core/components/knowledge/db/tests/test_knowledge_db.py

@@ -633,3 +633,18 @@ def _subjects(subject_type=None, obj='', predicate=None):
  assert _subjects(obj='linux') == {'infohash1', 'infohash2', 'infohash3'}
  assert _subjects(predicate=ResourceType.TAG, obj='linux') == {'infohash3'}
  assert _subjects(predicate=ResourceType.TITLE) == {'infohash1', 'infohash2'}
+
+ @db_session
+ def test_non_existent_misc(self):
+ """Test that get_misc returns proper values"""
+ # None if the key does not exist
+ assert not self.db.get_misc(key='non existent')
+
+ # A value if the key does exist
+ assert self.db.get_misc(key='non existent', default=42) == 42
+
+ @db_session
+ def test_set_misc(self):
+ """Test that set_misc works as expected"""
+ self.db.set_misc(key='key', value='value')
+ assert self.db.get_misc(key='key') == 'value'

src/tribler/core/components/knowledge/knowledge_component.py

@@ -4,7 +4,7 @@
 from tribler.core.components.key.key_component import KeyComponent
 from tribler.core.components.knowledge.community.knowledge_community import KnowledgeCommunity
 from tribler.core.components.knowledge.db.knowledge_db import KnowledgeDatabase
-from tribler.core.components.knowledge.rules.tag_rules_processor import KnowledgeRulesProcessor
+from tribler.core.components.knowledge.rules.knowledge_rules_processor import KnowledgeRulesProcessor
 from tribler.core.components.metadata_store.utils import generate_test_channels
 from tribler.core.utilities.simpledefs import STATEDIR_DB_DIR