diff --git a/src/pyipv8 b/src/pyipv8 index 00b9b44d6b..7bca86d1ef 160000 --- a/src/pyipv8 +++ b/src/pyipv8 @@ -1 +1 @@ -Subproject commit 00b9b44d6b83155cf124839d264fb986a02d5c56 +Subproject commit 7bca86d1ef09c7f39f407c6220e05d37e8395b81 diff --git a/src/tribler-common/tribler_common/tag_constants.py b/src/tribler-common/tribler_common/tag_constants.py new file mode 100644 index 0000000000..287ded9c4d --- /dev/null +++ b/src/tribler-common/tribler_common/tag_constants.py @@ -0,0 +1,2 @@ +MIN_TAG_LENGTH = 3 +MAX_TAG_LENGTH = 50 diff --git a/src/tribler-core/tribler_core/components/gigachannel/tests/test_gigachannel_component.py b/src/tribler-core/tribler_core/components/gigachannel/tests/test_gigachannel_component.py index 2d79f249ab..ae0e227721 100644 --- a/src/tribler-core/tribler_core/components/gigachannel/tests/test_gigachannel_component.py +++ b/src/tribler-core/tribler_core/components/gigachannel/tests/test_gigachannel_component.py @@ -6,13 +6,17 @@ from tribler_core.components.masterkey.masterkey_component import MasterKeyComponent from tribler_core.components.metadata_store.metadata_store_component import MetadataStoreComponent from tribler_core.components.restapi import RESTComponent +from tribler_core.components.tag.tag_component import TagComponent # pylint: disable=protected-access @pytest.mark.asyncio async def test_giga_channel_component(tribler_config): - components = [MetadataStoreComponent(), RESTComponent(), MasterKeyComponent(), Ipv8Component(), + tribler_config.ipv8.enabled = True + tribler_config.libtorrent.enabled = True + tribler_config.chant.enabled = True + components = [TagComponent(), MetadataStoreComponent(), RESTComponent(), MasterKeyComponent(), Ipv8Component(), GigaChannelComponent()] session = Session(tribler_config, components) with session: diff --git a/src/tribler-core/tribler_core/components/gigachannel_manager/tests/test_gigachannel_manager_component.py b/src/tribler-core/tribler_core/components/gigachannel_manager/tests/test_gigachannel_manager_component.py index ff951fe2d2..9506a5371c 100644 --- a/src/tribler-core/tribler_core/components/gigachannel_manager/tests/test_gigachannel_manager_component.py +++ b/src/tribler-core/tribler_core/components/gigachannel_manager/tests/test_gigachannel_manager_component.py @@ -2,18 +2,22 @@ from tribler_core.components.base import Session from tribler_core.components.gigachannel_manager.gigachannel_manager_component import GigachannelManagerComponent +from tribler_core.components.ipv8.ipv8_component import Ipv8Component from tribler_core.components.libtorrent.libtorrent_component import LibtorrentComponent from tribler_core.components.masterkey.masterkey_component import MasterKeyComponent from tribler_core.components.metadata_store.metadata_store_component import MetadataStoreComponent from tribler_core.components.restapi import RESTComponent from tribler_core.components.socks_servers.socks_servers_component import SocksServersComponent +from tribler_core.components.tag.tag_component import TagComponent # pylint: disable=protected-access + @pytest.mark.asyncio async def test_gigachannel_manager_component(tribler_config): - components = [SocksServersComponent(), MasterKeyComponent(), RESTComponent(), MetadataStoreComponent(), + components = [Ipv8Component(), TagComponent(), SocksServersComponent(), MasterKeyComponent(), RESTComponent(), + MetadataStoreComponent(), LibtorrentComponent(), GigachannelManagerComponent()] session = Session(tribler_config, components) with session: diff --git a/src/tribler-core/tribler_core/components/metadata_store/metadata_store_component.py b/src/tribler-core/tribler_core/components/metadata_store/metadata_store_component.py index 9e4095fcdc..e683a79856 100644 --- a/src/tribler-core/tribler_core/components/metadata_store/metadata_store_component.py +++ b/src/tribler-core/tribler_core/components/metadata_store/metadata_store_component.py @@ -4,6 +4,7 @@ from tribler_core.components.metadata_store.db.store import MetadataStore from tribler_core.components.metadata_store.utils import generate_test_channels from tribler_core.components.restapi import RestfulComponent +from tribler_core.components.tag.tag_component import TagComponent from tribler_core.components.upgrade import UpgradeComponent @@ -50,12 +51,17 @@ async def run(self): endpoints=['search', 'metadata', 'remote_query', 'downloads', 'channels', 'collections', 'statistics'], values={'mds': metadata_store} ) + tag_component = await self.require_component(TagComponent) + await self.init_endpoints( + endpoints=['channels', 'search'], + values={'tags_db': tag_component.tags_db} + ) self.session.notifier.add_observer(NTFY.TORRENT_METADATA_ADDED, metadata_store.TorrentMetadata.add_ffa_from_dict) if config.gui_test_mode: - generate_test_channels(metadata_store) + generate_test_channels(metadata_store, tag_component.tags_db) async def shutdown(self): await super().shutdown() diff --git a/src/tribler-core/tribler_core/components/metadata_store/restapi/channels_endpoint.py b/src/tribler-core/tribler_core/components/metadata_store/restapi/channels_endpoint.py index be9301762d..29b7975284 100644 --- a/src/tribler-core/tribler_core/components/metadata_store/restapi/channels_endpoint.py +++ b/src/tribler-core/tribler_core/components/metadata_store/restapi/channels_endpoint.py @@ -11,7 +11,7 @@ from ipv8.REST.schema import schema -from marshmallow.fields import Boolean, Dict, Integer, String +from marshmallow.fields import Boolean, Integer, String from pony.orm import db_session @@ -183,6 +183,7 @@ async def get_channel_contents(self, request): contents_list = [c.to_simple_dict() for c in contents] total = self.mds.get_total_count(**sanitized) if include_total else None self.add_download_progress_to_metadata_list(contents_list) + self.add_tags_to_metadata_list(contents_list, hide_xxx=sanitized["hide_xxx"]) response_dict = { "results": contents_list, "first": sanitized['first'], @@ -486,7 +487,7 @@ async def get_popular_torrents_channel(self, request): contents = self.mds.get_entries(**sanitized) contents_list = [c.to_simple_dict() for c in contents] self.add_download_progress_to_metadata_list(contents_list) - + self.add_tags_to_metadata_list(contents_list, hide_xxx=sanitized["hide_xxx"]) response_dict = { "results": contents_list, "first": sanitized['first'], diff --git a/src/tribler-core/tribler_core/components/metadata_store/restapi/metadata_endpoint_base.py b/src/tribler-core/tribler_core/components/metadata_store/restapi/metadata_endpoint_base.py index b30900f532..45ac8b5b95 100644 --- a/src/tribler-core/tribler_core/components/metadata_store/restapi/metadata_endpoint_base.py +++ b/src/tribler-core/tribler_core/components/metadata_store/restapi/metadata_endpoint_base.py @@ -1,4 +1,11 @@ +from binascii import unhexlify +from typing import Optional + +from pony.orm import db_session + +from tribler_core.components.metadata_store.category_filter.family_filter import default_xxx_filter from tribler_core.components.metadata_store.db.serialization import CHANNEL_TORRENT, COLLECTION_NODE, REGULAR_TORRENT +from tribler_core.components.tag.db.tag_db import TagDatabase from tribler_core.restapi.rest_endpoint import RESTEndpoint # This dict is used to translate JSON fields into the columns used in Pony for _sorting_. @@ -32,6 +39,7 @@ class MetadataEndpointBase(RESTEndpoint): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.mds = None + self.tags_db: Optional[TagDatabase] = None @classmethod def sanitize_parameters(cls, parameters): @@ -56,3 +64,12 @@ def sanitize_parameters(cls, parameters): mtypes.extend(metadata_type_to_search_scope[arg]) sanitized['metadata_type'] = frozenset(mtypes) return sanitized + + @db_session + def add_tags_to_metadata_list(self, contents_list, hide_xxx=False): + for torrent in contents_list: + if torrent['type'] == REGULAR_TORRENT: + tags = self.tags_db.get_tags(unhexlify(torrent["infohash"])) + if hide_xxx: + tags = [tag.lower() for tag in tags if not default_xxx_filter.isXXX(tag, isFilename=False)] + torrent["tags"] = tags diff --git a/src/tribler-core/tribler_core/components/metadata_store/restapi/search_endpoint.py b/src/tribler-core/tribler_core/components/metadata_store/restapi/search_endpoint.py index 6284cd0dbb..d23fd72c9d 100644 --- a/src/tribler-core/tribler_core/components/metadata_store/restapi/search_endpoint.py +++ b/src/tribler-core/tribler_core/components/metadata_store/restapi/search_endpoint.py @@ -4,12 +4,12 @@ from ipv8.REST.schema import schema -from marshmallow.fields import Boolean, Integer, String +from marshmallow.fields import Integer, String from pony.orm import db_session from tribler_core.components.metadata_store.restapi.metadata_endpoint import MetadataEndpointBase -from tribler_core.components.metadata_store.restapi.metadata_schema import MetadataParameters +from tribler_core.components.metadata_store.restapi.metadata_schema import MetadataParameters, MetadataSchema from tribler_core.components.metadata_store.db.store import MetadataStore from tribler_core.restapi.rest_endpoint import HTTP_BAD_REQUEST, RESTResponse from tribler_core.utilities.utilities import froze_it @@ -42,26 +42,12 @@ def sanitize_parameters(cls, parameters): 200: { 'schema': schema( SearchResponse={ - 'torrents': [ - schema( - Torrent={ - 'commit_status': Integer, - 'num_leechers': Integer, - 'date': Integer, - 'relevance_score': Integer, - 'id': Integer, - 'size': Integer, - 'category': String, - 'public_key': String, - 'name': String, - 'last_tracker_check': Integer, - 'infohash': String, - 'num_seeders': Integer, - 'type': String, - } - ) - ], - 'chant_dirty': Boolean, + 'results': [MetadataSchema], + 'first': Integer(), + 'last': Integer(), + 'sort_by': String(), + 'sort_desc': Integer(), + 'total': Integer(), } ) } @@ -98,6 +84,8 @@ def search_db(): self._logger.error("Error while performing DB search: %s: %s", type(e).__name__, e) return RESTResponse(status=HTTP_BAD_REQUEST) + self.add_tags_to_metadata_list(search_results, hide_xxx=sanitized["hide_xxx"]) + response_dict = { "results": search_results, "first": sanitized["first"], diff --git a/src/tribler-core/tribler_core/components/metadata_store/restapi/tests/conftest.py b/src/tribler-core/tribler_core/components/metadata_store/restapi/tests/conftest.py index b630f5ff18..174ed4c0a0 100644 --- a/src/tribler-core/tribler_core/components/metadata_store/restapi/tests/conftest.py +++ b/src/tribler-core/tribler_core/components/metadata_store/restapi/tests/conftest.py @@ -7,6 +7,7 @@ import pytest from tribler_core.components.metadata_store.db.orm_bindings.channel_node import NEW +from tribler_core.components.metadata_store.utils import tag_torrent from tribler_core.utilities.random_utils import random_infohash @@ -41,23 +42,34 @@ def add_fake_torrents_channels(metadata_store): @pytest.fixture -def my_channel(metadata_store): +def my_channel(metadata_store, tags_db): + """ + Generate a channel with some torrents. Also add a few (random) tags to these torrents. + """ with db_session: chan = metadata_store.ChannelMetadata.create_channel('test', 'test') for ind in range(5): + infohash = random_infohash() _ = metadata_store.TorrentMetadata( - origin_id=chan.id_, title='torrent%d' % ind, status=NEW, infohash=random_infohash() + origin_id=chan.id_, title='torrent%d' % ind, status=NEW, infohash=infohash ) + tag_torrent(infohash, tags_db) for ind in range(5, 9): - _ = metadata_store.TorrentMetadata(origin_id=chan.id_, title='torrent%d' % ind, infohash=random_infohash()) + infohash = random_infohash() + _ = metadata_store.TorrentMetadata(origin_id=chan.id_, title='torrent%d' % ind, infohash=infohash) + tag_torrent(infohash, tags_db) chan2 = metadata_store.ChannelMetadata.create_channel('test2', 'test2') for ind in range(5): + infohash = random_infohash() _ = metadata_store.TorrentMetadata( - origin_id=chan2.id_, title='torrentB%d' % ind, status=NEW, infohash=random_infohash() + origin_id=chan2.id_, title='torrentB%d' % ind, status=NEW, infohash=infohash ) + tag_torrent(infohash, tags_db) for ind in range(5, 9): + infohash = random_infohash() _ = metadata_store.TorrentMetadata( origin_id=chan2.id_, title='torrentB%d' % ind, infohash=random_infohash() ) + tag_torrent(infohash, tags_db) return chan diff --git a/src/tribler-core/tribler_core/components/metadata_store/restapi/tests/test_channels_endpoint.py b/src/tribler-core/tribler_core/components/metadata_store/restapi/tests/test_channels_endpoint.py index 1539702373..ce38d1d5b4 100644 --- a/src/tribler-core/tribler_core/components/metadata_store/restapi/tests/test_channels_endpoint.py +++ b/src/tribler-core/tribler_core/components/metadata_store/restapi/tests/test_channels_endpoint.py @@ -16,9 +16,11 @@ from tribler_core.components.libtorrent.torrentdef import TorrentDef from tribler_core.components.gigachannel.community.gigachannel_community import NoChannelSourcesException +from tribler_core.components.metadata_store.category_filter.family_filter import default_xxx_filter +from tribler_core.components.metadata_store.db.orm_bindings.channel_node import NEW from tribler_core.components.metadata_store.restapi.channels_endpoint import ChannelsEndpoint from tribler_core.components.metadata_store.db.serialization import CHANNEL_TORRENT, COLLECTION_NODE, REGULAR_TORRENT -from tribler_core.components.metadata_store.utils import RequestTimeoutException +from tribler_core.components.metadata_store.utils import RequestTimeoutException, tag_torrent from tribler_core.restapi.base_api_test import do_request from tribler_core.restapi.rest_manager import error_middleware from tribler_core.tests.tools.common import TORRENT_UBUNTU_FILE @@ -36,7 +38,7 @@ @pytest.fixture -def rest_api(loop, aiohttp_client, mock_dlmgr, metadata_store): # pylint: disable=unused-argument +def rest_api(loop, aiohttp_client, mock_dlmgr, metadata_store, tags_db): # pylint: disable=unused-argument mock_gigachannel_manager = Mock() mock_gigachannel_community = Mock() @@ -48,12 +50,14 @@ def return_exc(*args, **kwargs): mock_gigachannel_community.remote_select_channel_contents = return_exc collections_endpoint = ChannelsEndpoint() collections_endpoint.mds = metadata_store + collections_endpoint.tags_db = tags_db collections_endpoint.download_manager = mock_dlmgr collections_endpoint.gigachannel_manager = mock_gigachannel_manager collections_endpoint.gigachannel_community = mock_gigachannel_community channels_endpoint = ChannelsEndpoint() channels_endpoint.mds = metadata_store + channels_endpoint.tags_db = tags_db channels_endpoint.download_manager = mock_dlmgr channels_endpoint.gigachannel_manager = mock_gigachannel_manager channels_endpoint.gigachannel_community = mock_gigachannel_community @@ -700,3 +704,45 @@ async def test_get_channel_thumbnail(rest_api, metadata_store): assert response.status == 200 assert await response.read() == PNG_DATA assert response.headers["Content-Type"] == "image/png" + + +async def test_get_my_channel_tags(metadata_store, mock_dlmgr_get_download, my_channel, rest_api): # pylint: disable=redefined-outer-name + """ + Test whether tags are correctly returned over the REST API + """ + with db_session: + json_dict = await do_request( + rest_api, + 'channels/%s/%d?metadata_type=%d' + % (hexlify(my_channel.public_key), my_channel.id_, REGULAR_TORRENT), + expected_code=200, + ) + + assert len(json_dict['results']) == 9 + for item in json_dict['results']: + assert len(item["tags"]) >= 2 + + +async def test_get_my_channel_tags_xxx(metadata_store, tags_db, mock_dlmgr_get_download, my_channel, rest_api): # pylint: disable=redefined-outer-name + """ + Test whether XXX tags are correctly filtered + """ + with db_session: + chan = metadata_store.ChannelMetadata.create_channel('test', 'test') + infohash = random_infohash() + _ = metadata_store.TorrentMetadata(origin_id=chan.id_, title='taggedtorrent', status=NEW, infohash=infohash) + default_xxx_filter.xxx_terms = {"wrongterm"} + + # Add a few tags to our new torrent + tags = ["totally safe", "wrongterm", "wRonGtErM", "a wrongterm b"] + tag_torrent(infohash, tags_db, tags=tags) + + json_dict = await do_request( + rest_api, + 'channels/%s/%d?metadata_type=%d&hide_xxx=1' + % (hexlify(my_channel.public_key), chan.id_, REGULAR_TORRENT), + expected_code=200, + ) + + assert len(json_dict['results']) == 1 + assert len(json_dict['results'][0]["tags"]) == 1 diff --git a/src/tribler-core/tribler_core/components/metadata_store/restapi/tests/test_search_endpoint.py b/src/tribler-core/tribler_core/components/metadata_store/restapi/tests/test_search_endpoint.py index c108ccfbc8..f74791396c 100644 --- a/src/tribler-core/tribler_core/components/metadata_store/restapi/tests/test_search_endpoint.py +++ b/src/tribler-core/tribler_core/components/metadata_store/restapi/tests/test_search_endpoint.py @@ -9,8 +9,11 @@ from tribler_core.utilities.random_utils import random_infohash +# pylint: disable=unused-argument, redefined-outer-name + + @pytest.fixture -def needle_in_haystack_mds(metadata_store): # pylint: disable=unused-argument +def needle_in_haystack_mds(metadata_store): num_hay = 100 with db_session: _ = metadata_store.ChannelMetadata(title='test', tags='test', subscribed=True, infohash=random_infohash()) @@ -22,22 +25,23 @@ def needle_in_haystack_mds(metadata_store): # pylint: disable=unused-argument @pytest.fixture -def rest_api(loop, needle_in_haystack_mds, aiohttp_client): # pylint: disable=unused-argument +def rest_api(loop, needle_in_haystack_mds, aiohttp_client, tags_db): channels_endpoint = SearchEndpoint() channels_endpoint.mds = needle_in_haystack_mds + channels_endpoint.tags_db = tags_db app = Application() app.add_subapp('/search', channels_endpoint.app) return loop.run_until_complete(aiohttp_client(app)) -async def test_search_no_query(rest_api): # pylint: disable=unused-argument +async def test_search_no_query(rest_api): """ Testing whether the API returns an error 400 if no query is passed when doing a search """ await do_request(rest_api, 'search', expected_code=400) -async def test_search_wrong_mdtype(rest_api): # pylint: disable=unused-argument +async def test_search_wrong_mdtype(rest_api): """ Testing whether the API returns an error 400 if wrong metadata type is passed in the query """ @@ -106,14 +110,14 @@ async def test_search_with_include_total_and_max_rowid(rest_api): assert len(parsed["results"]) == 1 -async def test_completions_no_query(rest_api): # pylint: disable=unused-argument +async def test_completions_no_query(rest_api): """ Testing whether the API returns an error 400 if no query is passed when getting search completion terms """ await do_request(rest_api, 'search/completions', expected_code=400) -async def test_completions(rest_api): # pylint: disable=unused-argument +async def test_completions(rest_api): """ Testing whether the API returns the right terms when getting search completion terms """ diff --git a/src/tribler-core/tribler_core/components/metadata_store/tests/test_metadata_store_component.py b/src/tribler-core/tribler_core/components/metadata_store/tests/test_metadata_store_component.py index ff34d801b3..16cb293c1e 100644 --- a/src/tribler-core/tribler_core/components/metadata_store/tests/test_metadata_store_component.py +++ b/src/tribler-core/tribler_core/components/metadata_store/tests/test_metadata_store_component.py @@ -1,19 +1,18 @@ -from unittest.mock import patch - import pytest from tribler_core.components.base import Session +from tribler_core.components.ipv8.ipv8_component import Ipv8Component from tribler_core.components.masterkey.masterkey_component import MasterKeyComponent from tribler_core.components.metadata_store.metadata_store_component import MetadataStoreComponent from tribler_core.components.restapi import RESTComponent -from tribler_core.restapi.rest_manager import RESTManager +from tribler_core.components.tag.tag_component import TagComponent # pylint: disable=protected-access @pytest.mark.asyncio async def test_metadata_store_component(tribler_config): - components = [MasterKeyComponent(), RESTComponent(), MetadataStoreComponent()] + components = [TagComponent(), Ipv8Component(), MasterKeyComponent(), RESTComponent(), MetadataStoreComponent()] session = Session(tribler_config, components) with session: comp = MetadataStoreComponent.instance() diff --git a/src/tribler-core/tribler_core/components/metadata_store/utils.py b/src/tribler-core/tribler_core/components/metadata_store/utils.py index ed94ec9795..25ac3853be 100644 --- a/src/tribler-core/tribler_core/components/metadata_store/utils.py +++ b/src/tribler-core/tribler_core/components/metadata_store/utils.py @@ -1,12 +1,21 @@ import random +import string import time from ipv8.keyvault.crypto import default_eccrypto from pony.orm import db_session +from tribler_core.components.metadata_store.db.store import MetadataStore +from tribler_core.components.tag.community.tag_payload import TagOperation +from tribler_core.components.tag.db.tag_db import TagDatabase, TagOperationEnum from tribler_core.tests.tools.common import PNG_FILE -from tribler_core.utilities.random_utils import random_infohash, random_utf8_string +from tribler_core.utilities.random_utils import random_infohash, random_string, random_utf8_string + + +# Some random keys used for generating tags. +random_key_1 = default_eccrypto.generate_key('low') +random_key_2 = default_eccrypto.generate_key('low') class RequestTimeoutException(Exception): @@ -17,8 +26,21 @@ class NoChannelSourcesException(Exception): pass +def tag_torrent(infohash, tags_db, tags=None): + tags = tags or [random_string(size=random.randint(3, 10), chars=string.ascii_lowercase) + for _ in range(random.randint(2, 6))] + + # Give each torrent some tags + for tag in tags: + for key in [random_key_1, random_key_2]: # Each tag should be proposed by two unique users + counter = tags_db.get_next_operation_counter() + operation = TagOperation(infohash=infohash, tag=tag, operation=TagOperationEnum.ADD, timestamp=counter, + creator_public_key=key.pub().key_to_bin()) + tags_db.add_tag_operation(operation, b"") + + @db_session -def generate_torrent(metadata_store, parent): +def generate_torrent(metadata_store, tags_db, parent): infohash = random_infohash() # Give each torrent some health information. For now, we assume all torrents are healthy. @@ -28,16 +50,18 @@ def generate_torrent(metadata_store, parent): metadata_store.TorrentMetadata(title=random_utf8_string(50), infohash=infohash, origin_id=parent.id_, health=torrent_state) + tag_torrent(infohash, tags_db) + @db_session -def generate_collection(metadata_store, parent): +def generate_collection(metadata_store, tags_db, parent): coll = metadata_store.CollectionNode(title=random_utf8_string(50), origin_id=parent.id_) for _ in range(0, 3): - generate_torrent(metadata_store, coll) + generate_torrent(metadata_store, tags_db, coll) @db_session -def generate_channel(metadata_store, title=None, subscribed=False): +def generate_channel(metadata_store: MetadataStore, tags_db: TagDatabase, title=None, subscribed=False): # Remember and restore the original key orig_key = metadata_store.ChannelNode._my_key @@ -48,22 +72,22 @@ def generate_channel(metadata_store, title=None, subscribed=False): # add some collections to the channel for _ in range(0, 3): - generate_collection(metadata_store, chan) + generate_collection(metadata_store, tags_db, chan) metadata_store.ChannelNode._my_key = orig_key @db_session -def generate_test_channels(metadata_store): +def generate_test_channels(metadata_store, tags_db) -> None: # First, generate some foreign channels for ind in range(0, 10): - generate_channel(metadata_store, subscribed=ind % 2 == 0) + generate_channel(metadata_store, tags_db, subscribed=ind % 2 == 0) # This one is necessary to test filters, etc - generate_channel(metadata_store, title="non-random channel name") + generate_channel(metadata_store, tags_db, title="non-random channel name") # The same, but subscribed - generate_channel(metadata_store, title="non-random subscribed channel name", subscribed=True) + generate_channel(metadata_store, tags_db, title="non-random subscribed channel name", subscribed=True) # Now generate a couple of personal channels chan1 = metadata_store.ChannelMetadata.create_channel(title="personal channel with non-random name") @@ -74,12 +98,12 @@ def generate_test_channels(metadata_store): metadata_store.ChannelDescription(json_text='{"description_text": "# Hi guys"}', origin_id=chan1.id_) for _ in range(0, 3): - generate_collection(metadata_store, chan1) + generate_collection(metadata_store, tags_db, chan1) chan1.commit_channel_torrent() chan2 = metadata_store.ChannelMetadata.create_channel(title="personal channel " + random_utf8_string(50)) for _ in range(0, 3): - generate_collection(metadata_store, chan2) + generate_collection(metadata_store, tags_db, chan2) # add 'Tribler' entry to facilitate keyword search tests - generate_channel(metadata_store, title="Tribler tribler chan", subscribed=True) + generate_channel(metadata_store, tags_db, title="Tribler tribler chan", subscribed=True) diff --git a/src/tribler-core/tribler_core/components/popularity/tests/test_popularity_component.py b/src/tribler-core/tribler_core/components/popularity/tests/test_popularity_component.py index 44ac4b60b8..55daaf320c 100644 --- a/src/tribler-core/tribler_core/components/popularity/tests/test_popularity_component.py +++ b/src/tribler-core/tribler_core/components/popularity/tests/test_popularity_component.py @@ -8,6 +8,7 @@ from tribler_core.components.popularity.popularity_component import PopularityComponent from tribler_core.components.restapi import RESTComponent from tribler_core.components.socks_servers.socks_servers_component import SocksServersComponent +from tribler_core.components.tag.tag_component import TagComponent from tribler_core.components.torrent_checker import TorrentCheckerComponent @@ -16,8 +17,9 @@ @pytest.mark.asyncio async def test_popularity_component(tribler_config): - components = [SocksServersComponent(), LibtorrentComponent(), TorrentCheckerComponent(), MetadataStoreComponent(), - MasterKeyComponent(), RESTComponent(), Ipv8Component(), PopularityComponent()] + components = [SocksServersComponent(), LibtorrentComponent(), TorrentCheckerComponent(), TagComponent(), + MetadataStoreComponent(), MasterKeyComponent(), RESTComponent(), Ipv8Component(), + PopularityComponent()] session = Session(tribler_config, components) with session: await session.start() diff --git a/src/tribler-core/tribler_core/components/tag/__init__.py b/src/tribler-core/tribler_core/components/tag/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/tribler-core/tribler_core/components/tag/community/__init__.py b/src/tribler-core/tribler_core/components/tag/community/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/tribler-core/tribler_core/components/tag/community/tag_community.py b/src/tribler-core/tribler_core/components/tag/community/tag_community.py new file mode 100644 index 0000000000..108c3ea01c --- /dev/null +++ b/src/tribler-core/tribler_core/components/tag/community/tag_community.py @@ -0,0 +1,109 @@ +import random +from binascii import unhexlify + +from cryptography.exceptions import InvalidSignature +from pony.orm import db_session + +from ipv8.lazy_community import lazy_wrapper +from ipv8.types import Key +from tribler_core.components.tag.community.tag_payload import RawTagOperationMessage, RequestTagOperationMessage, \ + TagOperation, \ + TagOperationMessage, TagOperationSignature +from tribler_core.components.tag.community.tag_requests import PeerValidationError, TagRequests +from tribler_core.components.tag.db.tag_db import TagDatabase +from tribler_core.modules.tribler_community import TriblerCommunity + +REQUESTED_TAGS_COUNT = 10 + +REQUEST_INTERVAL = 5 # 5 sec +CLEAR_ALL_REQUESTS_INTERVAL = 10 * 60 # 10 minutes +TIME_DELTA_FOR_TAGS_THAT_READY_TO_GOSSIP = {'minutes': 1} + + +class TagCommunity(TriblerCommunity): + """ Community for disseminating tags across the network. + + Only tags are older than 1 minute will be gossiped. + """ + + community_id = unhexlify('042020c5e5e2ee0727fe99d704b430698e308d98') + + def __init__(self, *args, db: TagDatabase, request_interval=REQUEST_INTERVAL, + **kwargs): + super().__init__(*args, **kwargs) + self.db = db + self.requests = TagRequests() + + self.add_message_handler(RawTagOperationMessage, self.on_message) + self.add_message_handler(RequestTagOperationMessage, self.on_request) + + self.register_task("request_tags", self.request_tags, interval=request_interval) + self.register_task("clear_requests", self.requests.clear_requests, interval=CLEAR_ALL_REQUESTS_INTERVAL) + self.logger.info('Tag community initialized') + + def request_tags(self): + if not self.get_peers(): + return + + peer = random.choice(self.get_peers()) + self.requests.register_peer(peer, REQUESTED_TAGS_COUNT) + self.logger.debug(f'Request {REQUESTED_TAGS_COUNT} tags') + self.ez_send(peer, RequestTagOperationMessage(count=REQUESTED_TAGS_COUNT)) + + @lazy_wrapper(RawTagOperationMessage) + def on_message(self, peer, raw: RawTagOperationMessage): + self.logger.debug(f'Message received: {raw}') + operation, _ = self.serializer.unpack_serializable(TagOperation, raw.operation) + signature, _ = self.serializer.unpack_serializable(TagOperationSignature, raw.signature) + try: + remote_key = self.crypto.key_from_public_bin(operation.creator_public_key) + + self.requests.validate_peer(peer) + self.verify_signature(raw.operation, key=remote_key, signature=signature.signature) + operation.validate() + + with db_session(): + self.db.add_tag_operation(operation, signature.signature) + self.logger.info(f'Tag added: {operation.tag}:{operation.infohash}') + + except PeerValidationError as e: # peer has exhausted his response count + self.logger.warning(e) + except (ValueError, AssertionError) as e: # validation error + self.logger.warning(e) + except InvalidSignature as e: # signature verification error + self.logger.error(e) + + @lazy_wrapper(RequestTagOperationMessage) + def on_request(self, peer, operation): + tags_count = min(max(1, operation.count), REQUESTED_TAGS_COUNT) + self.logger.info(f'On request {tags_count} tags') + + with db_session: + random_tag_operations = self.db.get_tags_operations_for_gossip( + count=tags_count, + time_delta=TIME_DELTA_FOR_TAGS_THAT_READY_TO_GOSSIP + ) + + self.logger.debug(f'Response {len(random_tag_operations)} tags') + for tag_operation in random_tag_operations: + try: + operation = TagOperation( + infohash=tag_operation.torrent_tag.torrent.infohash, + operation=tag_operation.operation, + timestamp=tag_operation.timestamp, + creator_public_key=tag_operation.peer.public_key, + tag=tag_operation.torrent_tag.tag.name, + ) + operation.validate() + signature = TagOperationSignature(signature=tag_operation.signature) + self.ez_send(peer, TagOperationMessage(operation=operation, signature=signature)) + except (ValueError, AssertionError) as e: # validation error + self.logger.warning(e) + + def verify_signature(self, packed_message: bytes, key: Key, signature: bytes): + if not self.crypto.is_valid_signature(key, packed_message, signature): + raise InvalidSignature(f'Invalid signature for {packed_message}') + + def sign(self, operation: TagOperation) -> bytes: + packed = self.serializer.pack_serializable(operation) + return self.crypto.create_signature(self.my_peer.key, packed) diff --git a/src/tribler-core/tribler_core/components/tag/community/tag_payload.py b/src/tribler-core/tribler_core/components/tag/community/tag_payload.py new file mode 100644 index 0000000000..aa15a9e34a --- /dev/null +++ b/src/tribler-core/tribler_core/components/tag/community/tag_payload.py @@ -0,0 +1,59 @@ +from dataclasses import dataclass +from enum import IntEnum + +from ipv8.messaging.payload_dataclass import overwrite_dataclass, type_from_format +from tribler_common.tag_constants import MAX_TAG_LENGTH, MIN_TAG_LENGTH + +dataclass = overwrite_dataclass(dataclass) + + +class TagOperationEnum(IntEnum): + ADD = 1 + REMOVE = 2 + + +@dataclass +class TagOperation: + """Do not change the format of the TagOperationMessage, because this will result in an invalid signature. + """ + infohash: type_from_format('20s') + operation: int + timestamp: int + creator_public_key: type_from_format('74s') + tag: str + + def validate(self): + assert MIN_TAG_LENGTH <= len(self.tag) <= MAX_TAG_LENGTH, 'Tag length should be in range [3..50]' + assert not any(ch.isupper() for ch in self.tag), 'Tag should not contain upper-case letters' + + # try to convert operation into Enum + assert TagOperationEnum(self.operation) + + +RAW_DATA = type_from_format('varlenH') +TAG_OPERATION_MESSAGE_ID = 1 + + +@dataclass +class TagOperationSignature: + signature: type_from_format('64s') + + +@dataclass(msg_id=TAG_OPERATION_MESSAGE_ID) +class RawTagOperationMessage: + """ RAW payload class is used for reducing ipv8 unpacking operations + For more information take a look at: https://github.com/Tribler/tribler/pull/6396#discussion_r728334323 + """ + operation: RAW_DATA + signature: RAW_DATA + + +@dataclass(msg_id=TAG_OPERATION_MESSAGE_ID) +class TagOperationMessage: + operation: TagOperation + signature: TagOperationSignature + + +@dataclass(msg_id=2) +class RequestTagOperationMessage: + count: int diff --git a/src/tribler-core/tribler_core/components/tag/community/tag_requests.py b/src/tribler-core/tribler_core/components/tag/community/tag_requests.py new file mode 100644 index 0000000000..256bcc8d8a --- /dev/null +++ b/src/tribler-core/tribler_core/components/tag/community/tag_requests.py @@ -0,0 +1,30 @@ +from collections import defaultdict + + +class PeerValidationError(ValueError): + ... + + +class TagRequests: + """ This class is design for controlling requests during pull-based gossip. + + The main idea: + * Before a request, a client registered a peer with some number of expected responses + * While a response, the controller decrements number of expected responses for this peer + * The controller validates response by checking that expected responses for this peer is greater then 0 + """ + + def __init__(self): + self.requests = defaultdict(int) + + def register_peer(self, peer, number_of_responses): + self.requests[peer] = number_of_responses + + def validate_peer(self, peer): + if self.requests[peer] <= 0: + raise PeerValidationError(f'Peer has exhausted his response count {peer}') + + self.requests[peer] -= 1 + + def clear_requests(self): + self.requests = defaultdict(int) diff --git a/src/tribler-core/tribler_core/components/tag/community/tests/__init__.py b/src/tribler-core/tribler_core/components/tag/community/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/tribler-core/tribler_core/components/tag/community/tests/test_tag_community.py b/src/tribler-core/tribler_core/components/tag/community/tests/test_tag_community.py new file mode 100644 index 0000000000..97b72f8cdc --- /dev/null +++ b/src/tribler-core/tribler_core/components/tag/community/tests/test_tag_community.py @@ -0,0 +1,112 @@ +import datetime +from unittest.mock import MagicMock, Mock + +from cryptography.exceptions import InvalidSignature +from pony.orm import db_session + +from ipv8.test.base import TestBase +from ipv8.test.mocking.ipv8 import MockIPv8 +from tribler_core.components.tag.community.tag_community import TagCommunity +from tribler_core.components.tag.community.tag_payload import TagOperation +from tribler_core.components.tag.community.tag_requests import PeerValidationError +from tribler_core.components.tag.db.tag_db import TagDatabase, TagOperationEnum + +REQUEST_INTERVAL_FOR_RANDOM_TAGS = 0.1 # in seconds + + +class TestTagCommunity(TestBase): + def setUp(self): + super().setUp() + self.initialize(TagCommunity, 2) + + async def tearDown(self): + await super().tearDown() + + def create_node(self, *args, **kwargs): + return MockIPv8("curve25519", TagCommunity, db=TagDatabase(), request_interval=REQUEST_INTERVAL_FOR_RANDOM_TAGS) + + def create_message(self, tag=''): + community = self.overlay(0) + return TagOperation(infohash=b'1' * 20, + operation=TagOperationEnum.ADD, + timestamp=community.db.get_next_operation_counter(), + creator_public_key=community.my_peer.public_key.key_to_bin(), + tag=tag) + + @db_session + async def fill_db(self): + # create 10 tag operations: + # first 5 of them are correct + # next 5 of them are incorrect + community = self.overlay(0) + for i in range(10): + message = self.create_message(f'{i}' * 3) + signature = community.sign(message) + # 5 of them are signed incorrectly + if i >= 5: + signature = b'1' * 64 + + community.db.add_tag_operation(message, signature) + + # put them into the past + for tag_op in community.db.instance.TorrentTagOp.select(): + tag_op.set(updated_at=datetime.datetime.utcnow() - datetime.timedelta(minutes=2)) + + async def test_gossip(self): + # Test default gossip. + # Only 5 correct messages should be propagated + await self.fill_db() + await self.introduce_nodes() + await self.deliver_messages(timeout=REQUEST_INTERVAL_FOR_RANDOM_TAGS * 2) + with db_session: + assert self.overlay(0).db.instance.TorrentTagOp.select().count() == 10 + assert self.overlay(1).db.instance.TorrentTagOp.select().count() == 5 + + async def test_gossip_no_fresh_tags(self): + # Test that no fresh tags be propagated + # add one fresh operation into dataset and assert that it is not be propagated. + await self.fill_db() + + # put the first operation into the current moment (make it fresh) + with db_session: + tag_operation = self.overlay(0).db.instance.TorrentTagOp.select().first() + tag_operation.updated_at = datetime.datetime.utcnow() + + await self.introduce_nodes() + await self.deliver_messages(timeout=REQUEST_INTERVAL_FOR_RANDOM_TAGS * 2) + with db_session: + assert self.overlay(0).db.instance.TorrentTagOp.select().count() == 10 + assert self.overlay(1).db.instance.TorrentTagOp.select().count() == 4 # 5 invalid signature + 1 fresh tag + + async def test_on_message_eat_exceptions(self): + # Tests that except blocks in on_message function works as expected + # some exceptions should be eaten silently + exception_to_be_tested = {PeerValidationError, ValueError, AssertionError, + InvalidSignature} + await self.fill_db() + for exception_class in exception_to_be_tested: + # let's "break" the function that will be called on on_message() + self.overlay(1).verify_signature = Mock(side_effect=exception_class('')) + # occurred exception should be ate by community silently + await self.introduce_nodes() + await self.deliver_messages(timeout=REQUEST_INTERVAL_FOR_RANDOM_TAGS * 2) + self.overlay(1).verify_signature.assert_called() + + async def test_on_request_eat_exceptions(self): + # Tests that except blocks in on_request function works as expected + # ValueError should be eaten silently + await self.fill_db() + # let's "break" the function that will be called on on_request() + self.overlay(0).db.get_tags_operations_for_gossip = Mock(return_value=[MagicMock()]) + # occurred exception should be ate by community silently + await self.introduce_nodes() + await self.deliver_messages(timeout=REQUEST_INTERVAL_FOR_RANDOM_TAGS * 2) + self.overlay(0).db.get_tags_operations_for_gossip.assert_called() + + async def test_no_peers(self): + # Test that no error occurs in the community, in case there is no peers + self.overlay(0).get_peers = Mock(return_value=[]) + await self.fill_db() + await self.introduce_nodes() + await self.deliver_messages(timeout=REQUEST_INTERVAL_FOR_RANDOM_TAGS * 2) + self.overlay(0).get_peers.assert_called() diff --git a/src/tribler-core/tribler_core/components/tag/community/tests/test_tag_payload.py b/src/tribler-core/tribler_core/components/tag/community/tests/test_tag_payload.py new file mode 100644 index 0000000000..ff4ad9947b --- /dev/null +++ b/src/tribler-core/tribler_core/components/tag/community/tests/test_tag_payload.py @@ -0,0 +1,51 @@ +import pytest + +from tribler_core.components.tag.community.tag_payload import TagOperation +from tribler_core.components.tag.db.tag_db import TagOperationEnum + +pytestmark = pytest.mark.asyncio + + +def create_message(operation: int = TagOperationEnum.ADD, tag: str = 'tag') -> TagOperation: + return TagOperation(infohash=b'infohash', operation=operation, timestamp=0, + creator_public_key=b'creator_public_key', tag=tag) + + +async def test_correct_tag_size(): + create_message(tag='123').validate() + create_message(tag='1' * 50).validate() + + +async def test_empty_tag(): + with pytest.raises(AssertionError): + create_message(tag='').validate() + + +async def test_tag_less_than_3(): + with pytest.raises(AssertionError): + create_message(tag='12').validate() + + +async def test_tag_more_than_50(): + with pytest.raises(AssertionError): + create_message(tag='1' * 51).validate() + + +async def test_correct_operation(): + create_message(operation=TagOperationEnum.ADD).validate() + create_message(operation=1).validate() + + +async def test_incorrect_operation(): + with pytest.raises(ValueError): + create_message(operation=100).validate() + + +async def test_contain_upper_case(): + with pytest.raises(AssertionError): + create_message(tag='Tag').validate() + + +async def test_contain_upper_case_not_latin(): + with pytest.raises(AssertionError): + create_message(tag='Тэг').validate() diff --git a/src/tribler-core/tribler_core/components/tag/community/tests/test_tag_requests.py b/src/tribler-core/tribler_core/components/tag/community/tests/test_tag_requests.py new file mode 100644 index 0000000000..3082822b6d --- /dev/null +++ b/src/tribler-core/tribler_core/components/tag/community/tests/test_tag_requests.py @@ -0,0 +1,44 @@ +import pytest + +from tribler_core.components.tag.community.tag_requests import TagRequests + + +# pylint: disable=protected-access, redefined-outer-name + +@pytest.fixture +def tag_requests(): + return TagRequests() + + +pytestmark = pytest.mark.asyncio + + +async def test_add_peer(tag_requests): + tag_requests.register_peer('peer', number_of_responses=10) + assert tag_requests.requests['peer'] == 10 + + +async def test_clear_requests(tag_requests): + tag_requests.register_peer('peer', number_of_responses=10) + assert len(tag_requests.requests) == 1 + + tag_requests.clear_requests() + assert len(tag_requests.requests) == 0 + + +async def test_valid_peer(tag_requests): + tag_requests.register_peer('peer', number_of_responses=10) + tag_requests.validate_peer('peer') + + +async def test_missed_peer(tag_requests): + with pytest.raises(ValueError): + tag_requests.validate_peer('peer') + + +async def test_invalid_peer(tag_requests): + tag_requests.register_peer('peer', number_of_responses=1) + tag_requests.validate_peer('peer') + + with pytest.raises(ValueError): + tag_requests.validate_peer('peer') diff --git a/src/tribler-core/tribler_core/components/tag/db/tag_db.py b/src/tribler-core/tribler_core/components/tag/db/tag_db.py new file mode 100644 index 0000000000..24d531484f --- /dev/null +++ b/src/tribler-core/tribler_core/components/tag/db/tag_db.py @@ -0,0 +1,180 @@ +import datetime +import logging +from typing import List, Optional + +from pony import orm + +from tribler_core.components.tag.community.tag_payload import TagOperationEnum, TagOperation +from tribler_core.utilities.unicode import hexlify + + +class TagDatabase: + def __init__(self, filename: Optional[str] = None): + self.instance = orm.Database() + self.define_binding(self.instance) + self.instance.bind('sqlite', filename or ':memory:', create_db=True) + self.instance.generate_mapping(create_tables=True) + self.logger = logging.getLogger(self.__class__.__name__) + + @staticmethod + def define_binding(db): + class LocalPeer(db.Entity): # pylint: disable=unused-variable + counter = orm.Required(int, default=0, size=64) + + class Peer(db.Entity): + id = orm.PrimaryKey(int, auto=True) + public_key = orm.Required(bytes, unique=True) + added_at = orm.Optional(datetime.datetime, default=datetime.datetime.utcnow) + operations = orm.Set(lambda: TorrentTagOp) + + class Torrent(db.Entity): + id = orm.PrimaryKey(int, auto=True) + infohash = orm.Required(bytes, unique=True) + tags = orm.Set(lambda: TorrentTag) + + class TorrentTag(db.Entity): + id = orm.PrimaryKey(int, auto=True) + torrent = orm.Required(lambda: Torrent) + tag = orm.Required(lambda: Tag) + operations = orm.Set(lambda: TorrentTagOp) + + added_count = orm.Required(int, default=0) + removed_count = orm.Required(int, default=0) + + local_operation = orm.Optional(int) # in case user don't (or do) want to see it locally + + orm.composite_key(torrent, tag) + + def update_counter(self, operation: TagOperationEnum, increment: int = 1, is_local_peer: bool = False): + """ Update TorrentTag's counter + Args: + operation: Tag operation + increment: + is_local_peer: The flag indicates whether do we performs operations from a local user or from + a remote user. In case of the local user, his operations will be considered as + authoritative for his (only) local Tribler instance. + + Returns: + """ + if is_local_peer: + self.local_operation = operation + if operation == TagOperationEnum.ADD: + self.added_count += increment + if operation == TagOperationEnum.REMOVE: + self.removed_count += increment + + class Tag(db.Entity): + id = orm.PrimaryKey(int, auto=True) + name = orm.Required(str, unique=True) + torrents = orm.Set(lambda: TorrentTag) + + class TorrentTagOp(db.Entity): + id = orm.PrimaryKey(int, auto=True) + + torrent_tag = orm.Required(lambda: TorrentTag) + peer = orm.Required(lambda: Peer) + + operation = orm.Required(int) + timestamp = orm.Required(int) + signature = orm.Required(bytes) + updated_at = orm.Required(datetime.datetime, default=datetime.datetime.utcnow) + + orm.composite_key(torrent_tag, peer) + + @staticmethod + def _get_or_create(cls, create_kwargs=None, **kwargs): # pylint: disable=bad-staticmethod-argument + """Get or create db entity. + Args: + cls: Entity's class, eg: `self.instance.Peer` + create_kwargs: Additional arguments for creating new entity + **kwargs: Arguments for selecting or for creating in case of missing entity + + Returns: Entity's instance + """ + obj = cls.get_for_update(**kwargs) + if not obj: + if create_kwargs: + kwargs.update(create_kwargs) + obj = cls(**kwargs) + return obj + + def add_tag_operation(self, operation: TagOperation, signature: bytes, is_local_peer: bool = False): + """ Add the operation that will be applied to the tag. + Args: + operation: the class describes the adding operation + signature: the signature of the operation + is_local_peer: local operations processes differently than remote operations. They affects + `TorrentTag.local_operation` field which is used in `self.get_tags()` function. + + Returns: + """ + self.logger.debug(f'Add tag operation. Infohash: {hexlify(operation.infohash)}, tag: {operation.tag}') + peer = self._get_or_create(self.instance.Peer, public_key=operation.creator_public_key) + tag = self._get_or_create(self.instance.Tag, name=operation.tag) + torrent = self._get_or_create(self.instance.Torrent, infohash=operation.infohash) + torrent_tag = self._get_or_create(self.instance.TorrentTag, tag=tag, torrent=torrent) + op = self.instance.TorrentTagOp.get_for_update(torrent_tag=torrent_tag, peer=peer) + + if not op: # then insert + self.instance.TorrentTagOp(torrent_tag=torrent_tag, peer=peer, operation=operation.operation, + timestamp=operation.timestamp, signature=signature) + torrent_tag.update_counter(operation.operation, is_local_peer=is_local_peer) + return + + # if it is a message from the past, then return + if operation.timestamp <= op.timestamp: + return + + # To prevent endless incrementing of the operation, we apply the following logic: + + # 1. Decrement previous operation + torrent_tag.update_counter(op.operation, increment=-1, is_local_peer=is_local_peer) + # 2. Increment new operation + torrent_tag.update_counter(operation.operation, is_local_peer=is_local_peer) + # 3. Update the operation entity + op.set(operation=operation.operation, timestamp=operation.timestamp, signature=signature, + updated_at=datetime.datetime.utcnow()) + + def get_tags(self, infohash: bytes) -> List[str]: + """ Get all tags for this particular torrent. + + Returns: A list of tags + """ + self.logger.debug(f'Get tags. Infohash: {hexlify(infohash)}') + + torrent = self.instance.Torrent.get(infohash=infohash) + if not torrent: + return [] + + def show_condition(torrent_tag): + return torrent_tag.local_operation == TagOperationEnum.ADD.value or \ + not torrent_tag.local_operation and torrent_tag.added_count >= 2 + + query = torrent.tags.select(show_condition) + query = orm.select(tt.tag.name for tt in query) + return list(query) + + def get_next_operation_counter(self) -> int: + """ Get counter of last operation and increment this counter in DB. + Returns: Counter that represented by integer (starts from 1). + """ + local_peer = self._get_or_create(self.instance.LocalPeer) + next_operation_counter = local_peer.counter + 1 + local_peer.set(counter=next_operation_counter) + + return next_operation_counter + + def get_tags_operations_for_gossip(self, time_delta, count: int = 10) -> List: + """ Get random operations from the DB that older than time_delta. + + Args: + time_delta: a dictionary for `datetime.timedelta` + count: a limit for a resulting query + """ + updated_at = datetime.datetime.utcnow() - datetime.timedelta(**time_delta) + return list(self.instance.TorrentTagOp + .select(lambda tto: tto.updated_at <= updated_at) + .random(count)) + + def shutdown(self) -> None: + self.instance.disconnect() diff --git a/src/tribler-core/tribler_core/components/tag/db/tests/test_tag_db.py b/src/tribler-core/tribler_core/components/tag/db/tests/test_tag_db.py new file mode 100644 index 0000000000..2a4b490d9a --- /dev/null +++ b/src/tribler-core/tribler_core/components/tag/db/tests/test_tag_db.py @@ -0,0 +1,246 @@ +import datetime + +from pony.orm import commit, db_session + +from ipv8.test.base import TestBase +from tribler_core.components.tag.community.tag_payload import TagOperationEnum, TagOperation +from tribler_core.components.tag.db.tag_db import TagDatabase + + +# pylint: disable=protected-access + + +class TestTagDB(TestBase): + def setUp(self): + super().setUp() + self.db = TagDatabase() + + async def tearDown(self): + await super().tearDown() + + def create_torrent_tag(self, tag='tag', infohash=b'infohash'): + tag = self.db._get_or_create(self.db.instance.Tag, name=tag) + torrent = self.db._get_or_create(self.db.instance.Torrent, infohash=infohash) + torrent_tag = self.db._get_or_create(self.db.instance.TorrentTag, tag=tag, torrent=torrent) + + return torrent_tag + + def add_operation(self, infohash=b'', tag='', peer=b'', operation=TagOperationEnum.ADD, + is_local_peer=False, timestamp=None): + timestamp = timestamp or self.db.get_next_operation_counter() + message = TagOperation(infohash=infohash, tag=tag, operation=operation, + timestamp=timestamp, creator_public_key=peer) + self.db.add_tag_operation(message, signature=b'', is_local_peer=is_local_peer) + commit() + + @db_session + async def test_get_or_create(self): + # Test that function get_or_create() works as expected: + # it gets an entity if the entity is exist and create the entity otherwise + assert self.db.instance.Peer.select().count() == 0 + + # test create + peer = self.db._get_or_create(self.db.instance.Peer, public_key=b'123') + commit() + assert peer.public_key == b'123' + assert self.db.instance.Peer.select().count() == 1 + + # test get + peer = self.db._get_or_create(self.db.instance.Peer, public_key=b'123') + assert peer.public_key == b'123' + assert self.db.instance.Peer.select().count() == 1 + + @db_session + async def test_update_counter_add(self): + torrent_tag = self.create_torrent_tag() + + # let's update ADD counter + torrent_tag.update_counter(TagOperationEnum.ADD, increment=1) + assert torrent_tag.added_count == 1 + assert torrent_tag.removed_count == 0 + assert not torrent_tag.local_operation + + @db_session + async def test_update_counter_remove(self): + torrent_tag = self.create_torrent_tag() + + # let's update REMOVE counter + torrent_tag.update_counter(TagOperationEnum.REMOVE, increment=1) + assert torrent_tag.added_count == 0 + assert torrent_tag.removed_count == 1 + assert not torrent_tag.local_operation + + @db_session + async def test_update_counter_Local(self): + torrent_tag = self.create_torrent_tag() + + # let's update local operation + torrent_tag.update_counter(TagOperationEnum.REMOVE, increment=1, is_local_peer=True) + assert torrent_tag.added_count == 0 + assert torrent_tag.removed_count == 1 + assert torrent_tag.local_operation == TagOperationEnum.REMOVE + + @db_session + async def test_remote_add_tag_operation(self): + def assert_all_tables_have_the_only_one_entity(): + assert self.db.instance.Peer.select().count() == 1 + assert self.db.instance.Torrent.select().count() == 1 + assert self.db.instance.TorrentTag.select().count() == 1 + assert self.db.instance.Tag.select().count() == 1 + assert self.db.instance.TorrentTagOp.select().count() == 1 + + # add the first operation + self.add_operation(b'infohash', 'tag', b'peer1') + assert_all_tables_have_the_only_one_entity() + + # add the same operation + self.add_operation(b'infohash', 'tag', b'peer1') + assert_all_tables_have_the_only_one_entity() + + # add an operation from the past + self.add_operation(b'infohash', 'tag', b'peer1', timestamp=0) + assert_all_tables_have_the_only_one_entity() + + # add a duplicate operation but from the future + self.add_operation(b'infohash', 'tag', b'peer1', timestamp=1000) + assert_all_tables_have_the_only_one_entity() + + assert self.db.instance.TorrentTagOp.get().operation == TagOperationEnum.ADD + assert self.db.instance.TorrentTag.get().added_count == 1 + assert self.db.instance.TorrentTag.get().removed_count == 0 + + # add a unique operation from the future + self.add_operation(b'infohash', 'tag', b'peer1', operation=TagOperationEnum.REMOVE, timestamp=1001) + assert_all_tables_have_the_only_one_entity() + assert self.db.instance.TorrentTagOp.get().operation == TagOperationEnum.REMOVE + assert self.db.instance.TorrentTag.get().added_count == 0 + assert self.db.instance.TorrentTag.get().removed_count == 1 + + @db_session + async def test_remote_add_multiple_tag_operations(self): + self.add_operation(b'infohash', 'tag', b'peer1') + self.add_operation(b'infohash', 'tag', b'peer2') + self.add_operation(b'infohash', 'tag', b'peer3') + + assert self.db.instance.TorrentTag.get().added_count == 3 + assert self.db.instance.TorrentTag.get().removed_count == 0 + + self.add_operation(b'infohash', 'tag', b'peer2', operation=TagOperationEnum.REMOVE) + assert self.db.instance.TorrentTag.get().added_count == 2 + assert self.db.instance.TorrentTag.get().removed_count == 1 + + self.add_operation(b'infohash', 'tag', b'peer1', operation=TagOperationEnum.REMOVE) + assert self.db.instance.TorrentTag.get().added_count == 1 + assert self.db.instance.TorrentTag.get().removed_count == 2 + + self.add_operation(b'infohash', 'tag', b'peer1') + assert self.db.instance.TorrentTag.get().added_count == 2 + assert self.db.instance.TorrentTag.get().removed_count == 1 + + @db_session + async def test_multiple_tags(self): + # peer1 + self.add_operation(b'infohash1', 'tag1', b'peer1') + self.add_operation(b'infohash1', 'tag2', b'peer1') + self.add_operation(b'infohash1', 'tag3', b'peer1') + + self.add_operation(b'infohash2', 'tag4', b'peer1') + self.add_operation(b'infohash2', 'tag5', b'peer1') + self.add_operation(b'infohash2', 'tag6', b'peer1') + + # peer2 + self.add_operation(b'infohash1', 'tag1', b'peer2') + self.add_operation(b'infohash1', 'tag2', b'peer2') + + # peer3 + self.add_operation(b'infohash2', 'tag1', b'peer3') + self.add_operation(b'infohash2', 'tag2', b'peer3') + + def assert_entities_count(): + assert self.db.instance.Peer.select().count() == 3 + assert self.db.instance.Torrent.select().count() == 2 + assert self.db.instance.TorrentTag.select().count() == 8 + assert self.db.instance.Tag.select().count() == 6 + assert self.db.instance.TorrentTagOp.select().count() == 10 + + assert_entities_count() + + torrent1 = self.db.instance.Torrent.get(infohash=b'infohash1') + tag1 = self.db.instance.Tag.get(name='tag1') + torrent_tag = self.db.instance.TorrentTag.get(torrent=torrent1, tag=tag1) + assert torrent_tag.added_count == 2 + assert torrent_tag.removed_count == 0 + + self.add_operation(b'infohash1', 'tag1', b'peer2', operation=TagOperationEnum.REMOVE) + self.add_operation(b'infohash1', 'tag2', b'peer2', operation=TagOperationEnum.REMOVE) + assert_entities_count() + assert torrent_tag.added_count == 1 + assert torrent_tag.removed_count == 1 + + @db_session + async def test_get_tags(self): + # Test that only tags above a threshold (2) is shown + + # peer1 + self.add_operation(b'infohash1', 'tag1', b'peer1') + self.add_operation(b'infohash1', 'tag2', b'peer1') + self.add_operation(b'infohash1', 'tag3', b'peer1') + + self.add_operation(b'infohash2', 'tag4', b'peer1') + self.add_operation(b'infohash2', 'tag5', b'peer1') + self.add_operation(b'infohash2', 'tag6', b'peer1') + + # peer2 + self.add_operation(b'infohash1', 'tag1', b'peer2') + self.add_operation(b'infohash1', 'tag2', b'peer2') + + # peer3 + self.add_operation(b'infohash2', 'tag1', b'peer3') + self.add_operation(b'infohash2', 'tag2', b'peer3') + + assert self.db.get_tags(b'infohash1') == ['tag1', 'tag2'] + assert self.db.get_tags(b'infohash2') == [] + assert self.db.get_tags(b'infohash3') == [] + + @db_session + async def test_show_local_tags(self): + # Test that locally added tags have a priority to show. + # That means no matter of other peers opinions, locally added tag should be visible. + self.add_operation(b'infohash1', 'tag1', b'peer1', operation=TagOperationEnum.REMOVE) + self.add_operation(b'infohash1', 'tag1', b'peer2', operation=TagOperationEnum.REMOVE) + assert not self.db.get_tags(b'infohash1') + + # test local add + self.add_operation(b'infohash1', 'tag1', b'peer3', operation=TagOperationEnum.ADD, is_local_peer=True) + assert self.db.get_tags(b'infohash1') == ['tag1'] + + @db_session + async def test_hide_local_tags(self): + # Test that locally removed tags should not be visible to local user. + # No matter of other peers opinions, locally removed tag should be not visible. + self.add_operation(b'infohash1', 'tag1', b'peer1') + self.add_operation(b'infohash1', 'tag1', b'peer2') + assert self.db.get_tags(b'infohash1') == ['tag1'] + + # test local remove + self.add_operation(b'infohash1', 'tag1', b'peer3', operation=TagOperationEnum.REMOVE, is_local_peer=True) + assert self.db.get_tags(b'infohash1') == [] + + @db_session + async def test_get_next_operation_counter(self): + assert self.db.get_next_operation_counter() == 1 + assert self.db.get_next_operation_counter() == 2 + + @db_session + async def test_get_tags_operations_for_gossip(self): + time_delta = {'minutes': 1} + self.add_operation(b'infohash1', 'tag1', b'peer1') + self.add_operation(b'infohash1', 'tag2', b'peer1') + # assert that immediately added torrents are not returned + assert not self.db.get_tags_operations_for_gossip(time_delta) + + tag_operation = self.db.instance.TorrentTagOp.select().first() + tag_operation.updated_at = datetime.datetime.utcnow() - datetime.timedelta(minutes=2) + + # assert that only one torrent returned (the old one) + assert len(self.db.get_tags_operations_for_gossip(time_delta)) == 1 diff --git a/src/tribler-core/tribler_core/components/tag/restapi/tags_endpoint.py b/src/tribler-core/tribler_core/components/tag/restapi/tags_endpoint.py new file mode 100644 index 0000000000..1809476491 --- /dev/null +++ b/src/tribler-core/tribler_core/components/tag/restapi/tags_endpoint.py @@ -0,0 +1,91 @@ +import binascii +from binascii import unhexlify +from typing import Optional, Set + +from aiohttp import web +from aiohttp_apispec import docs +from marshmallow.fields import Boolean +from pony.orm import db_session + +from ipv8.REST.schema import schema +from tribler_common.tag_constants import MAX_TAG_LENGTH, MIN_TAG_LENGTH +from tribler_core.components.tag.community.tag_community import TagCommunity +from tribler_core.components.tag.community.tag_payload import TagOperation +from tribler_core.components.tag.db.tag_db import TagDatabase, TagOperationEnum +from tribler_core.restapi.rest_endpoint import HTTP_BAD_REQUEST, RESTEndpoint, RESTResponse +from tribler_core.restapi.schema import HandledErrorSchema +from tribler_core.utilities.utilities import froze_it + + +@froze_it +class TagsEndpoint(RESTEndpoint): + """ + Top-level endpoint for tags. + """ + + def __init__(self, *args, **kwargs): + RESTEndpoint.__init__(self, *args, **kwargs) + self.db: Optional[TagDatabase] = None + self.community: Optional[TagCommunity] = None + + def setup_routes(self): + self.app.add_routes( + [ + web.patch('/{infohash}', self.update_tags_entries), + ] + ) + + @docs( + tags=["General"], + summary="Update a particular torrent with tags.", + responses={ + 200: { + "schema": schema(UpdateTagsResponse={'success': Boolean()}) + }, + HTTP_BAD_REQUEST: { + "schema": HandledErrorSchema, 'example': {"error": "Invalid tag length"}}, + }, + description="This endpoint updates a particular torrent with the provided tags." + ) + async def update_tags_entries(self, request): + params = await request.json() + try: + infohash = unhexlify(request.match_info['infohash']) + if len(infohash) != 20: + return RESTResponse({"error": "Invalid infohash"}, status=HTTP_BAD_REQUEST) + except binascii.Error: + return RESTResponse({"error": "Invalid infohash"}, status=HTTP_BAD_REQUEST) + + tags = {tag.lower() for tag in params["tags"]} + + # Validate whether the size of the tag is within the allowed range + for tag in tags: + if len(tag) < MIN_TAG_LENGTH or len(tag) > MAX_TAG_LENGTH: + return RESTResponse({"error": "Invalid tag length"}, status=HTTP_BAD_REQUEST) + + self.modify_tags(infohash, tags) + + return RESTResponse({"success": True}) + + @db_session + def modify_tags(self, infohash: bytes, new_tags: Set[str]): + """ + Modify the tags of a particular content item. + """ + if not self.db or not self.community: + return + + # First, get the current tags and compute the diff between the old and new tags + old_tags = set(self.db.get_tags(infohash)) + added_tags = new_tags - old_tags + removed_tags = old_tags - new_tags + + # Create individual tag operations for the added/removed tags + public_key = self.community.my_peer.key.pub().key_to_bin() + for tag in added_tags.union(removed_tags): + operation = TagOperationEnum.ADD if tag in added_tags else TagOperationEnum.REMOVE + counter = self.db.get_next_operation_counter() + operation = TagOperation(infohash=infohash, operation=operation, timestamp=counter, + creator_public_key=public_key, tag=tag) + signature = self.community.sign(operation) + self.db.add_tag_operation(operation, signature, is_local_peer=True) diff --git a/src/tribler-core/tribler_core/components/tag/restapi/tests/test_tags_endpoint.py b/src/tribler-core/tribler_core/components/tag/restapi/tests/test_tags_endpoint.py new file mode 100644 index 0000000000..87e59c6bc1 --- /dev/null +++ b/src/tribler-core/tribler_core/components/tag/restapi/tests/test_tags_endpoint.py @@ -0,0 +1,87 @@ +from unittest.mock import Mock + +import pytest +from aiohttp.web_app import Application +from freezegun import freeze_time +from pony.orm import db_session + +from tribler_core.components.tag.restapi.tags_endpoint import TagsEndpoint +from tribler_core.conftest import TEST_PERSONAL_KEY +from tribler_core.restapi.base_api_test import do_request +from tribler_core.utilities.unicode import hexlify + + +# pylint: disable=redefined-outer-name + +@pytest.fixture +def tags_endpoint(tags_db): + endpoint = TagsEndpoint() + endpoint.db = tags_db + endpoint.community = Mock() + endpoint.community.my_peer.key = TEST_PERSONAL_KEY + endpoint.community.sign = Mock(return_value=b'') + return endpoint + + +@pytest.fixture +def rest_api(loop, aiohttp_client, tags_endpoint): + app = Application() + app.add_subapp('/tags', tags_endpoint.app) + return loop.run_until_complete(aiohttp_client(app)) + + +async def test_add_tag_invalid_infohash(rest_api): + """ + Test whether an error is returned if we try to add a tag to content with an invalid infohash + """ + post_data = {"tags": ["abc", "def"]} + await do_request(rest_api, 'tags/3f3', request_type="PATCH", expected_code=400, post_data=post_data) + await do_request(rest_api, 'tags/3f3f', request_type="PATCH", expected_code=400, post_data=post_data) + + +async def test_add_invalid_tag(rest_api): + """ + Test whether an error is returned if we try to add a tag that is too short or long. + """ + post_data = {"tags": ["a"]} + infohash = b'a' * 20 + await do_request(rest_api, f'tags/{hexlify(infohash)}', request_type="PATCH", expected_code=400, + post_data=post_data) + + post_data = {"tags": ["a" * 60]} + await do_request(rest_api, f'tags/{hexlify(infohash)}', request_type="PATCH", expected_code=400, + post_data=post_data) + + +async def test_modify_tags(rest_api, tags_db): + """ + Test modifying tags + """ + post_data = {"tags": ["abc", "def"]} + infohash = b'a' * 20 + with freeze_time("2015-01-01") as frozen_time: + await do_request(rest_api, f'tags/{hexlify(infohash)}', request_type="PATCH", expected_code=200, + post_data=post_data) + with db_session: + tags = tags_db.get_tags(infohash) + assert len(tags) == 2 + + # Now remove a tag + frozen_time.move_to("2016-01-01") + post_data = {"tags": ["abc"]} + await do_request(rest_api, f'tags/{hexlify(infohash)}', request_type="PATCH", expected_code=200, + post_data=post_data) + with db_session: + tags = tags_db.get_tags(infohash) + assert tags == ["abc"] + + +async def test_modify_tags_no_community(tags_db, tags_endpoint): + tags_endpoint.community = None + infohash = b'a' * 20 + tags_endpoint.modify_tags(infohash, {"abc", "def"}) + + with db_session: + tags = tags_db.get_tags(infohash) + + assert len(tags) == 0 diff --git a/src/tribler-core/tribler_core/components/tag/tag_component.py b/src/tribler-core/tribler_core/components/tag/tag_component.py new file mode 100644 index 0000000000..a1255d6de6 --- /dev/null +++ b/src/tribler-core/tribler_core/components/tag/tag_component.py @@ -0,0 +1,47 @@ +from tribler_common.simpledefs import STATEDIR_DB_DIR +from tribler_core.components.ipv8.ipv8_component import Ipv8Component + +from tribler_core.components.restapi import RestfulComponent +from tribler_core.components.tag.community.tag_community import TagCommunity +from tribler_core.components.tag.db.tag_db import TagDatabase + + +class TagComponent(RestfulComponent): + community: TagCommunity = None + tags_db: TagDatabase = None + _ipv8_component: Ipv8Component = None + + async def run(self): + await super().run() + + self._ipv8_component = await self.require_component(Ipv8Component) + + db_name = "tags_gui_test.db" if self.session.config.gui_test_mode else "tags.db" + db_path = self.session.config.state_dir / STATEDIR_DB_DIR / db_name + + if self.session.config.gui_test_mode and db_path.exists(): + # Make sure that we start with a clean metadata database when in GUI mode every time. + self.logger.info("Wiping tags database in GUI test mode") + db_path.unlink(missing_ok=True) + + self.tags_db = TagDatabase(str(db_path)) + self.community = TagCommunity( + self._ipv8_component.peer, + self._ipv8_component.ipv8.endpoint, + self._ipv8_component.ipv8.network, + db=self.tags_db, + ) + + await self.init_endpoints( + endpoints=['tags'], + values={'db': self.tags_db, 'community': self.community} + ) + + self._ipv8_component.initialise_community_by_default(self.community) + + async def shutdown(self): + await super().shutdown() + if self._ipv8_component and self.community: + await self._ipv8_component.unload_community(self.community) + if self.tags_db: + self.tags_db.shutdown() diff --git a/src/tribler-core/tribler_core/components/tag/tests/test_tag_component.py b/src/tribler-core/tribler_core/components/tag/tests/test_tag_component.py new file mode 100644 index 0000000000..c28e2ef9e2 --- /dev/null +++ b/src/tribler-core/tribler_core/components/tag/tests/test_tag_component.py @@ -0,0 +1,24 @@ +import pytest + +from tribler_core.components.base import Session +from tribler_core.components.ipv8.ipv8_component import Ipv8Component +from tribler_core.components.masterkey.masterkey_component import MasterKeyComponent +from tribler_core.components.restapi import RESTComponent +from tribler_core.components.tag.tag_component import TagComponent + +# pylint: disable=protected-access + +COMPONENTS = [] + + +@pytest.mark.asyncio +async def test_tag_component(tribler_config): + session = Session(tribler_config, [MasterKeyComponent(), Ipv8Component(), RESTComponent(), TagComponent()]) + with session: + comp = TagComponent.instance() + await session.start() + + assert comp.started_event.is_set() and not comp.failed + assert comp.community + + await session.shutdown() diff --git a/src/tribler-core/tribler_core/components/tests/test_tribler_components.py b/src/tribler-core/tribler_core/components/tests/test_tribler_components.py index 591af2a616..5d2d9152e4 100644 --- a/src/tribler-core/tribler_core/components/tests/test_tribler_components.py +++ b/src/tribler-core/tribler_core/components/tests/test_tribler_components.py @@ -7,6 +7,7 @@ from tribler_core.components.metadata_store.metadata_store_component import MetadataStoreComponent from tribler_core.components.restapi import RESTComponent from tribler_core.components.socks_servers.socks_servers_component import SocksServersComponent +from tribler_core.components.tag.tag_component import TagComponent from tribler_core.components.torrent_checker import TorrentCheckerComponent from tribler_core.components.tunnels import TunnelsComponent from tribler_core.components.upgrade import UpgradeComponent @@ -57,7 +58,7 @@ async def test_REST_component(tribler_config): async def test_torrent_checker_component(tribler_config): components = [SocksServersComponent(), LibtorrentComponent(), MasterKeyComponent(), RESTComponent(), - MetadataStoreComponent(), TorrentCheckerComponent()] + Ipv8Component(), TagComponent(), MetadataStoreComponent(), TorrentCheckerComponent()] session = Session(tribler_config, components) with session: await session.start() diff --git a/src/tribler-core/tribler_core/conftest.py b/src/tribler-core/tribler_core/conftest.py index 59a761a4c3..a47103d55f 100644 --- a/src/tribler-core/tribler_core/conftest.py +++ b/src/tribler-core/tribler_core/conftest.py @@ -12,6 +12,7 @@ from tribler_common.network_utils import NetworkUtils from tribler_common.simpledefs import DLSTATUS_SEEDING +from tribler_core.components.tag.db.tag_db import TagDatabase from tribler_core.config.tribler_config import TriblerConfig from tribler_core.components.libtorrent.download_manager.download import Download @@ -196,6 +197,13 @@ def metadata_store(tmp_path): mds.shutdown() +@pytest.fixture +def tags_db(): + db = TagDatabase() + yield db + db.shutdown() + + @pytest.fixture def dispersy_to_pony_migrator(metadata_store): dispersy_db_path = TESTS_DATA_DIR / 'upgrade_databases/tribler_v29.sdb' diff --git a/src/tribler-core/tribler_core/restapi/root_endpoint.py b/src/tribler-core/tribler_core/restapi/root_endpoint.py index f71f83ea03..68814337d1 100644 --- a/src/tribler-core/tribler_core/restapi/root_endpoint.py +++ b/src/tribler-core/tribler_core/restapi/root_endpoint.py @@ -1,4 +1,5 @@ from ipv8.REST.root_endpoint import RootEndpoint as IPV8RootEndpoint +from tribler_core.components.tag.restapi.tags_endpoint import TagsEndpoint from tribler_core.config.tribler_config import TriblerConfig from tribler_core.components.bandwidth_accounting.restapi.bandwidth_endpoint import BandwidthEndpoint @@ -58,7 +59,8 @@ def setup_routes(self): '/collections': (ChannelsEndpoint, self.tribler_config.chant.enabled), '/search': (SearchEndpoint, self.tribler_config.chant.enabled), '/remote_query': (RemoteQueryEndpoint, self.tribler_config.chant.enabled), - '/ipv8': (IPV8RootEndpoint, self.tribler_config.ipv8.enabled) + '/ipv8': (IPV8RootEndpoint, self.tribler_config.ipv8.enabled), + '/tags': (TagsEndpoint, self.tribler_config.chant.enabled), } for path, (ep_cls, enabled) in endpoints.items(): if enabled: diff --git a/src/tribler-core/tribler_core/start_core.py b/src/tribler-core/tribler_core/start_core.py index 3fc6dc661e..84107b5fee 100644 --- a/src/tribler-core/tribler_core/start_core.py +++ b/src/tribler-core/tribler_core/start_core.py @@ -27,6 +27,7 @@ from tribler_core.components.resource_monitor.resource_monitor_component import ResourceMonitorComponent from tribler_core.components.restapi import RESTComponent from tribler_core.components.socks_servers.socks_servers_component import SocksServersComponent +from tribler_core.components.tag.tag_component import TagComponent from tribler_core.components.torrent_checker import TorrentCheckerComponent from tribler_core.components.tunnels import TunnelsComponent from tribler_core.components.upgrade import UpgradeComponent @@ -53,7 +54,10 @@ def components_gen(config: TriblerConfig): yield MetadataStoreComponent() if config.ipv8.enabled: yield Ipv8Component() + yield MasterKeyComponent() + yield TagComponent() + if config.libtorrent.enabled: yield SocksServersComponent() if config.libtorrent.enabled: diff --git a/src/tribler-gui/tribler_gui/tests/test_gui.py b/src/tribler-gui/tribler_gui/tests/test_gui.py index 96b078daf5..a737cad3a0 100644 --- a/src/tribler-gui/tribler_gui/tests/test_gui.py +++ b/src/tribler-gui/tribler_gui/tests/test_gui.py @@ -360,7 +360,7 @@ def test_search_suggestions(window): @pytest.mark.guitest def test_search(window): - window.top_search_bar.setText("trib") + window.top_search_bar.setText("a") # This is likely to trigger some search results QTest.keyClick(window.top_search_bar, Qt.Key_Enter) wait_for_variable(window, "search_results_page.search_request") screenshot(window, name="search_loading_page")