Merge pull request #6396 from drew2a/feature/tag_community

Adding tag community, database and REST API endpoints

src/tribler-common/tribler_common/tag_constants.py

@@ -0,0 +1,2 @@
+MIN_TAG_LENGTH = 3
+MAX_TAG_LENGTH = 50

src/tribler-core/tribler_core/components/gigachannel/tests/test_gigachannel_component.py

@@ -6,13 +6,17 @@
 from tribler_core.components.masterkey.masterkey_component import MasterKeyComponent
 from tribler_core.components.metadata_store.metadata_store_component import MetadataStoreComponent
 from tribler_core.components.restapi import RESTComponent
+from tribler_core.components.tag.tag_component import TagComponent
 
 
 # pylint: disable=protected-access
 
 @pytest.mark.asyncio
 async def test_giga_channel_component(tribler_config):
-    components = [MetadataStoreComponent(), RESTComponent(), MasterKeyComponent(), Ipv8Component(),
+    tribler_config.ipv8.enabled = True
+    tribler_config.libtorrent.enabled = True
+    tribler_config.chant.enabled = True
+    components = [TagComponent(), MetadataStoreComponent(), RESTComponent(), MasterKeyComponent(), Ipv8Component(),
                   GigaChannelComponent()]
     session = Session(tribler_config, components)
     with session:

src/tribler-core/tribler_core/components/gigachannel_manager/tests/test_gigachannel_manager_component.py

@@ -2,18 +2,22 @@
 
 from tribler_core.components.base import Session
 from tribler_core.components.gigachannel_manager.gigachannel_manager_component import GigachannelManagerComponent
+from tribler_core.components.ipv8.ipv8_component import Ipv8Component
 from tribler_core.components.libtorrent.libtorrent_component import LibtorrentComponent
 from tribler_core.components.masterkey.masterkey_component import MasterKeyComponent
 from tribler_core.components.metadata_store.metadata_store_component import MetadataStoreComponent
 from tribler_core.components.restapi import RESTComponent
 from tribler_core.components.socks_servers.socks_servers_component import SocksServersComponent
+from tribler_core.components.tag.tag_component import TagComponent
 
 
 # pylint: disable=protected-access
 
+
 @pytest.mark.asyncio
 async def test_gigachannel_manager_component(tribler_config):
-    components = [SocksServersComponent(), MasterKeyComponent(), RESTComponent(), MetadataStoreComponent(),
+    components = [Ipv8Component(), TagComponent(), SocksServersComponent(), MasterKeyComponent(), RESTComponent(),
+                  MetadataStoreComponent(),
                   LibtorrentComponent(), GigachannelManagerComponent()]
     session = Session(tribler_config, components)
     with session:

src/tribler-core/tribler_core/components/metadata_store/metadata_store_component.py

@@ -4,6 +4,7 @@
 from tribler_core.components.metadata_store.db.store import MetadataStore
 from tribler_core.components.metadata_store.utils import generate_test_channels
 from tribler_core.components.restapi import RestfulComponent
+from tribler_core.components.tag.tag_component import TagComponent
 from tribler_core.components.upgrade import UpgradeComponent
 
 
@@ -50,12 +51,17 @@ async def run(self):
             endpoints=['search', 'metadata', 'remote_query', 'downloads', 'channels', 'collections', 'statistics'],
             values={'mds': metadata_store}
         )
+        tag_component = await self.require_component(TagComponent)
+        await self.init_endpoints(
+            endpoints=['channels', 'search'],
+            values={'tags_db': tag_component.tags_db}
+        )
 
         self.session.notifier.add_observer(NTFY.TORRENT_METADATA_ADDED,
                                            metadata_store.TorrentMetadata.add_ffa_from_dict)
 
         if config.gui_test_mode:
-            generate_test_channels(metadata_store)
+            generate_test_channels(metadata_store, tag_component.tags_db)
 
     async def shutdown(self):
         await super().shutdown()

src/tribler-core/tribler_core/components/metadata_store/restapi/channels_endpoint.py

@@ -11,7 +11,7 @@
 
 from ipv8.REST.schema import schema
 
-from marshmallow.fields import Boolean, Dict, Integer, String
+from marshmallow.fields import Boolean, Integer, String
 
 from pony.orm import db_session
 
@@ -183,6 +183,7 @@ async def get_channel_contents(self, request):
                 contents_list = [c.to_simple_dict() for c in contents]
                 total = self.mds.get_total_count(**sanitized) if include_total else None
         self.add_download_progress_to_metadata_list(contents_list)
+        self.add_tags_to_metadata_list(contents_list, hide_xxx=sanitized["hide_xxx"])
         response_dict = {
             "results": contents_list,
             "first": sanitized['first'],
@@ -486,7 +487,7 @@ async def get_popular_torrents_channel(self, request):
             contents = self.mds.get_entries(**sanitized)
             contents_list = [c.to_simple_dict() for c in contents]
         self.add_download_progress_to_metadata_list(contents_list)
-
+        self.add_tags_to_metadata_list(contents_list, hide_xxx=sanitized["hide_xxx"])
         response_dict = {
             "results": contents_list,
             "first": sanitized['first'],

src/tribler-core/tribler_core/components/metadata_store/restapi/metadata_endpoint_base.py

@@ -1,4 +1,11 @@
+from binascii import unhexlify
+from typing import Optional
+
+from pony.orm import db_session
+
+from tribler_core.components.metadata_store.category_filter.family_filter import default_xxx_filter
 from tribler_core.components.metadata_store.db.serialization import CHANNEL_TORRENT, COLLECTION_NODE, REGULAR_TORRENT
+from tribler_core.components.tag.db.tag_db import TagDatabase
 from tribler_core.restapi.rest_endpoint import RESTEndpoint
 
 # This dict is used to translate JSON fields into the columns used in Pony for _sorting_.
@@ -32,6 +39,7 @@ class MetadataEndpointBase(RESTEndpoint):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.mds = None
+        self.tags_db: Optional[TagDatabase] = None
 
     @classmethod
     def sanitize_parameters(cls, parameters):
@@ -56,3 +64,12 @@ def sanitize_parameters(cls, parameters):
                 mtypes.extend(metadata_type_to_search_scope[arg])
             sanitized['metadata_type'] = frozenset(mtypes)
         return sanitized
+
+    @db_session
+    def add_tags_to_metadata_list(self, contents_list, hide_xxx=False):
+        for torrent in contents_list:
+            if torrent['type'] == REGULAR_TORRENT:
+                tags = self.tags_db.get_tags(unhexlify(torrent["infohash"]))
+                if hide_xxx:
+                    tags = [tag.lower() for tag in tags if not default_xxx_filter.isXXX(tag, isFilename=False)]
+                torrent["tags"] = tags

src/tribler-core/tribler_core/components/metadata_store/restapi/search_endpoint.py

@@ -4,12 +4,12 @@
 
 from ipv8.REST.schema import schema
 
-from marshmallow.fields import Boolean, Integer, String
+from marshmallow.fields import Integer, String
 
 from pony.orm import db_session
 
 from tribler_core.components.metadata_store.restapi.metadata_endpoint import MetadataEndpointBase
-from tribler_core.components.metadata_store.restapi.metadata_schema import MetadataParameters
+from tribler_core.components.metadata_store.restapi.metadata_schema import MetadataParameters, MetadataSchema
 from tribler_core.components.metadata_store.db.store import MetadataStore
 from tribler_core.restapi.rest_endpoint import HTTP_BAD_REQUEST, RESTResponse
 from tribler_core.utilities.utilities import froze_it
@@ -42,26 +42,12 @@ def sanitize_parameters(cls, parameters):
             200: {
                 'schema': schema(
                     SearchResponse={
-                        'torrents': [
-                            schema(
-                                Torrent={
-                                    'commit_status': Integer,
-                                    'num_leechers': Integer,
-                                    'date': Integer,
-                                    'relevance_score': Integer,
-                                    'id': Integer,
-                                    'size': Integer,
-                                    'category': String,
-                                    'public_key': String,
-                                    'name': String,
-                                    'last_tracker_check': Integer,
-                                    'infohash': String,
-                                    'num_seeders': Integer,
-                                    'type': String,
-                                }
-                            )
-                        ],
-                        'chant_dirty': Boolean,
+                        'results': [MetadataSchema],
+                        'first': Integer(),
+                        'last': Integer(),
+                        'sort_by': String(),
+                        'sort_desc': Integer(),
+                        'total': Integer(),
                     }
                 )
             }
@@ -98,6 +84,8 @@ def search_db():
             self._logger.error("Error while performing DB search: %s: %s", type(e).__name__, e)
             return RESTResponse(status=HTTP_BAD_REQUEST)
 
+        self.add_tags_to_metadata_list(search_results, hide_xxx=sanitized["hide_xxx"])
+
         response_dict = {
             "results": search_results,
             "first": sanitized["first"],

src/tribler-core/tribler_core/components/metadata_store/restapi/tests/conftest.py

@@ -7,6 +7,7 @@
 import pytest
 
 from tribler_core.components.metadata_store.db.orm_bindings.channel_node import NEW
+from tribler_core.components.metadata_store.utils import tag_torrent
 from tribler_core.utilities.random_utils import random_infohash
 
 
@@ -41,23 +42,34 @@ def add_fake_torrents_channels(metadata_store):
 
 
 @pytest.fixture
-def my_channel(metadata_store):
+def my_channel(metadata_store, tags_db):
+    """
+    Generate a channel with some torrents. Also add a few (random) tags to these torrents.
+    """
     with db_session:
         chan = metadata_store.ChannelMetadata.create_channel('test', 'test')
         for ind in range(5):
+            infohash = random_infohash()
             _ = metadata_store.TorrentMetadata(
-                origin_id=chan.id_, title='torrent%d' % ind, status=NEW, infohash=random_infohash()
+                origin_id=chan.id_, title='torrent%d' % ind, status=NEW, infohash=infohash
             )
+            tag_torrent(infohash, tags_db)
         for ind in range(5, 9):
-            _ = metadata_store.TorrentMetadata(origin_id=chan.id_, title='torrent%d' % ind, infohash=random_infohash())
+            infohash = random_infohash()
+            _ = metadata_store.TorrentMetadata(origin_id=chan.id_, title='torrent%d' % ind, infohash=infohash)
+            tag_torrent(infohash, tags_db)
 
         chan2 = metadata_store.ChannelMetadata.create_channel('test2', 'test2')
         for ind in range(5):
+            infohash = random_infohash()
             _ = metadata_store.TorrentMetadata(
-                origin_id=chan2.id_, title='torrentB%d' % ind, status=NEW, infohash=random_infohash()
+                origin_id=chan2.id_, title='torrentB%d' % ind, status=NEW, infohash=infohash
             )
+            tag_torrent(infohash, tags_db)
         for ind in range(5, 9):
+            infohash = random_infohash()
             _ = metadata_store.TorrentMetadata(
                 origin_id=chan2.id_, title='torrentB%d' % ind, infohash=random_infohash()
             )
+            tag_torrent(infohash, tags_db)
         return chan

src/tribler-core/tribler_core/components/metadata_store/restapi/tests/test_channels_endpoint.py

@@ -16,9 +16,11 @@
 
 from tribler_core.components.libtorrent.torrentdef import TorrentDef
 from tribler_core.components.gigachannel.community.gigachannel_community import NoChannelSourcesException
+from tribler_core.components.metadata_store.category_filter.family_filter import default_xxx_filter
+from tribler_core.components.metadata_store.db.orm_bindings.channel_node import NEW
 from tribler_core.components.metadata_store.restapi.channels_endpoint import ChannelsEndpoint
 from tribler_core.components.metadata_store.db.serialization import CHANNEL_TORRENT, COLLECTION_NODE, REGULAR_TORRENT
-from tribler_core.components.metadata_store.utils import RequestTimeoutException
+from tribler_core.components.metadata_store.utils import RequestTimeoutException, tag_torrent
 from tribler_core.restapi.base_api_test import do_request
 from tribler_core.restapi.rest_manager import error_middleware
 from tribler_core.tests.tools.common import TORRENT_UBUNTU_FILE
@@ -36,7 +38,7 @@
 
 
 @pytest.fixture
-def rest_api(loop, aiohttp_client, mock_dlmgr, metadata_store):  # pylint: disable=unused-argument
+def rest_api(loop, aiohttp_client, mock_dlmgr, metadata_store, tags_db):  # pylint: disable=unused-argument
     mock_gigachannel_manager = Mock()
     mock_gigachannel_community = Mock()
 
@@ -48,12 +50,14 @@ def return_exc(*args, **kwargs):
     mock_gigachannel_community.remote_select_channel_contents = return_exc
     collections_endpoint = ChannelsEndpoint()
     collections_endpoint.mds = metadata_store
+    collections_endpoint.tags_db = tags_db
     collections_endpoint.download_manager = mock_dlmgr
     collections_endpoint.gigachannel_manager = mock_gigachannel_manager
     collections_endpoint.gigachannel_community = mock_gigachannel_community
 
     channels_endpoint = ChannelsEndpoint()
     channels_endpoint.mds = metadata_store
+    channels_endpoint.tags_db = tags_db
     channels_endpoint.download_manager = mock_dlmgr
     channels_endpoint.gigachannel_manager = mock_gigachannel_manager
     channels_endpoint.gigachannel_community = mock_gigachannel_community
@@ -700,3 +704,45 @@ async def test_get_channel_thumbnail(rest_api, metadata_store):
             assert response.status == 200
             assert await response.read() == PNG_DATA
             assert response.headers["Content-Type"] == "image/png"
+
+
+async def test_get_my_channel_tags(metadata_store, mock_dlmgr_get_download, my_channel, rest_api):  # pylint: disable=redefined-outer-name
+    """
+    Test whether tags are correctly returned over the REST API
+    """
+    with db_session:
+        json_dict = await do_request(
+            rest_api,
+            'channels/%s/%d?metadata_type=%d'
+            % (hexlify(my_channel.public_key), my_channel.id_, REGULAR_TORRENT),
+            expected_code=200,
+        )
+
+    assert len(json_dict['results']) == 9
+    for item in json_dict['results']:
+        assert len(item["tags"]) >= 2
+
+
+async def test_get_my_channel_tags_xxx(metadata_store, tags_db, mock_dlmgr_get_download, my_channel, rest_api):  # pylint: disable=redefined-outer-name
+    """
+    Test whether XXX tags are correctly filtered
+    """
+    with db_session:
+        chan = metadata_store.ChannelMetadata.create_channel('test', 'test')
+        infohash = random_infohash()
+        _ = metadata_store.TorrentMetadata(origin_id=chan.id_, title='taggedtorrent', status=NEW, infohash=infohash)
+        default_xxx_filter.xxx_terms = {"wrongterm"}
+
+        # Add a few tags to our new torrent
+        tags = ["totally safe", "wrongterm", "wRonGtErM", "a wrongterm b"]
+        tag_torrent(infohash, tags_db, tags=tags)
+
+        json_dict = await do_request(
+            rest_api,
+            'channels/%s/%d?metadata_type=%d&hide_xxx=1'
+            % (hexlify(my_channel.public_key), chan.id_, REGULAR_TORRENT),
+            expected_code=200,
+        )
+
+    assert len(json_dict['results']) == 1
+    assert len(json_dict['results'][0]["tags"]) == 1

src/tribler-core/tribler_core/components/metadata_store/restapi/tests/test_search_endpoint.py

@@ -9,8 +9,11 @@
 from tribler_core.utilities.random_utils import random_infohash
 
 
+# pylint: disable=unused-argument, redefined-outer-name
+
+
 @pytest.fixture
-def needle_in_haystack_mds(metadata_store):  # pylint: disable=unused-argument
+def needle_in_haystack_mds(metadata_store):
     num_hay = 100
     with db_session:
         _ = metadata_store.ChannelMetadata(title='test', tags='test', subscribed=True, infohash=random_infohash())
@@ -22,22 +25,23 @@ def needle_in_haystack_mds(metadata_store):  # pylint: disable=unused-argument
 
 
 @pytest.fixture
-def rest_api(loop, needle_in_haystack_mds, aiohttp_client):  # pylint: disable=unused-argument
+def rest_api(loop, needle_in_haystack_mds, aiohttp_client, tags_db):
     channels_endpoint = SearchEndpoint()
     channels_endpoint.mds = needle_in_haystack_mds
+    channels_endpoint.tags_db = tags_db
     app = Application()
     app.add_subapp('/search', channels_endpoint.app)
     return loop.run_until_complete(aiohttp_client(app))
 
 
-async def test_search_no_query(rest_api):  # pylint: disable=unused-argument
+async def test_search_no_query(rest_api):
     """
     Testing whether the API returns an error 400 if no query is passed when doing a search
     """
     await do_request(rest_api, 'search', expected_code=400)
 
 
-async def test_search_wrong_mdtype(rest_api):  # pylint: disable=unused-argument
+async def test_search_wrong_mdtype(rest_api):
     """
     Testing whether the API returns an error 400 if wrong metadata type is passed in the query
     """
@@ -106,14 +110,14 @@ async def test_search_with_include_total_and_max_rowid(rest_api):
     assert len(parsed["results"]) == 1
 
 
-async def test_completions_no_query(rest_api):  # pylint: disable=unused-argument
+async def test_completions_no_query(rest_api):
     """
     Testing whether the API returns an error 400 if no query is passed when getting search completion terms
     """
     await do_request(rest_api, 'search/completions', expected_code=400)
 
 
-async def test_completions(rest_api):  # pylint: disable=unused-argument
+async def test_completions(rest_api):
     """
     Testing whether the API returns the right terms when getting search completion terms
     """