diff --git a/.flake8 b/.flake8 index 5a5da222a47..a36f40f6b51 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,4 @@ [flake8] -jobs = auto max-line-length = 120 select=E901,E999,F821,F822,F823,I100,I101,I201,I202,T001,T002,T003,T004 ignore=E203,W503,C0330 diff --git a/src/tribler-core/tribler_core/conftest.py b/src/tribler-core/tribler_core/conftest.py index 12c350d11ec..77450ff57d4 100644 --- a/src/tribler-core/tribler_core/conftest.py +++ b/src/tribler-core/tribler_core/conftest.py @@ -2,15 +2,18 @@ from pathlib import Path from unittest.mock import Mock -import pytest from aiohttp import web -from pony.orm import db_session -from ipv8.database import database_blob from ipv8.keyvault.private.libnaclkey import LibNaCLSK from ipv8.util import succeed + +from pony.orm import db_session + +import pytest + from tribler_common.network_utils import NetworkUtils from tribler_common.simpledefs import DLSTATUS_SEEDING + from tribler_core.config.tribler_config import TriblerConfig from tribler_core.modules.libtorrent.download import Download from tribler_core.modules.libtorrent.download_config import DownloadConfig @@ -313,6 +316,6 @@ def needle_in_haystack(enable_chant, enable_api, session): # pylint: disable=un _ = session.mds.ChannelMetadata(title='test', tags='test', subscribed=True, infohash=random_infohash()) for x in range(0, num_hay): session.mds.TorrentMetadata(title='hay ' + str(x), infohash=random_infohash()) - session.mds.TorrentMetadata(title='needle', infohash=database_blob(bytearray(random_infohash()))) - session.mds.TorrentMetadata(title='needle2', infohash=database_blob(bytearray(random_infohash()))) + session.mds.TorrentMetadata(title='needle', infohash=random_infohash()) + session.mds.TorrentMetadata(title='needle2', infohash=random_infohash()) return session diff --git a/src/tribler-core/tribler_core/modules/bandwidth_accounting/transaction.py b/src/tribler-core/tribler_core/modules/bandwidth_accounting/transaction.py index 7639f681265..3b0a4d23bde 100644 --- a/src/tribler-core/tribler_core/modules/bandwidth_accounting/transaction.py +++ b/src/tribler-core/tribler_core/modules/bandwidth_accounting/transaction.py @@ -39,7 +39,7 @@ def pack(self, signature_a=True, signature_b=True) -> bytes: Encode this block for transport. :param signature_a: False to pack EMPTY_SIG in the location of signature A, true to pack the signature A field. :param signature_b: False to pack EMPTY_SIG in the location of signature B, true to pack the signature B field. - :return: the database_blob the data was packed into. + :return: bytes object the data was packed into. """ args = [self.sequence_number, self.public_key_a, self.public_key_b, self.signature_a if signature_a else EMPTY_SIGNATURE, diff --git a/src/tribler-core/tribler_core/modules/metadata_store/community/tests/test_gigachannel_community.py b/src/tribler-core/tribler_core/modules/metadata_store/community/tests/test_gigachannel_community.py index 48c22d87c01..016a9db785d 100644 --- a/src/tribler-core/tribler_core/modules/metadata_store/community/tests/test_gigachannel_community.py +++ b/src/tribler-core/tribler_core/modules/metadata_store/community/tests/test_gigachannel_community.py @@ -3,7 +3,6 @@ from unittest import mock from unittest.mock import Mock, PropertyMock, patch -from ipv8.database import database_blob from ipv8.keyvault.crypto import default_eccrypto from ipv8.peer import Peer from ipv8.test.base import TestBase @@ -25,7 +24,7 @@ from tribler_core.utilities.path_util import Path from tribler_core.utilities.random_utils import random_infohash -EMPTY_BLOB = database_blob(b"") +EMPTY_BLOB = b"" # pylint:disable=protected-access diff --git a/src/tribler-core/tribler_core/modules/metadata_store/gigachannel_manager.py b/src/tribler-core/tribler_core/modules/metadata_store/gigachannel_manager.py index e6c8bea14c9..b476424b44c 100644 --- a/src/tribler-core/tribler_core/modules/metadata_store/gigachannel_manager.py +++ b/src/tribler-core/tribler_core/modules/metadata_store/gigachannel_manager.py @@ -1,7 +1,6 @@ import asyncio from asyncio import CancelledError, wait_for -from ipv8.database import database_blob from ipv8.taskmanager import TaskManager, task from pony.orm import db_session @@ -310,7 +309,7 @@ def updated_my_channel(self, tdef): Notify the core that we updated our channel. """ with db_session: - my_channel = self.session.mds.ChannelMetadata.get(infohash=database_blob(tdef.get_infohash())) + my_channel = self.session.mds.ChannelMetadata.get(infohash=tdef.get_infohash()) if ( my_channel and my_channel.status == COMMITTED diff --git a/src/tribler-core/tribler_core/modules/metadata_store/orm_bindings/channel_metadata.py b/src/tribler-core/tribler_core/modules/metadata_store/orm_bindings/channel_metadata.py index ef8b6cdfbbc..899a3778877 100644 --- a/src/tribler-core/tribler_core/modules/metadata_store/orm_bindings/channel_metadata.py +++ b/src/tribler-core/tribler_core/modules/metadata_store/orm_bindings/channel_metadata.py @@ -2,8 +2,6 @@ from binascii import unhexlify from datetime import datetime -from ipv8.database import database_blob - from lz4.frame import LZ4FrameCompressor from pony import orm @@ -246,7 +244,7 @@ class ChannelMetadata(db.TorrentMetadata, db.CollectionNode): @db_session def get_my_channels(cls): return ChannelMetadata.select( - lambda g: g.origin_id == 0 and g.public_key == database_blob(cls._my_key.pub().key_to_bin()[10:]) + lambda g: g.origin_id == 0 and g.public_key == cls._my_key.pub().key_to_bin()[10:] ) @classmethod @@ -261,7 +259,7 @@ def create_channel(cls, title, description="", origin_id=0): """ my_channel = cls( origin_id=origin_id, - public_key=database_blob(cls._my_key.pub().key_to_bin()[10:]), + public_key=cls._my_key.pub().key_to_bin()[10:], title=title, tags=description, subscribed=True, @@ -447,15 +445,12 @@ def get_channels_by_title(cls, title): @classmethod @db_session def get_channel_with_infohash(cls, infohash): - return cls.get(infohash=database_blob(infohash)) + return cls.get(infohash=infohash) @classmethod @db_session def get_recent_channel_with_public_key(cls, public_key): - return ( - cls.select(lambda g: g.public_key == database_blob(public_key)).sort_by(lambda g: desc(g.id_)).first() - or None - ) + return cls.select(lambda g: g.public_key == public_key).sort_by(lambda g: desc(g.id_)).first() or None @classmethod @db_session @@ -487,7 +482,7 @@ def get_updated_channels(cls): if g.subscribed == 1 and g.status != LEGACY_ENTRY and (g.local_version < g.timestamp) - and g.public_key != database_blob(cls._my_key.pub().key_to_bin()[10:]) + and g.public_key != cls._my_key.pub().key_to_bin()[10:] ) # don't simplify `g.subscribed == 1` to bool form, it is used by partial index! @property @@ -553,7 +548,7 @@ def get_channel_name(cls, dl_name, infohash): if not channel: return dl_name - if channel.infohash == database_blob(infohash): + if channel.infohash == infohash: return channel.title return 'OLD:' + channel.title diff --git a/src/tribler-core/tribler_core/modules/metadata_store/orm_bindings/channel_node.py b/src/tribler-core/tribler_core/modules/metadata_store/orm_bindings/channel_node.py index a2437872bf2..aaed7b6dd96 100644 --- a/src/tribler-core/tribler_core/modules/metadata_store/orm_bindings/channel_node.py +++ b/src/tribler-core/tribler_core/modules/metadata_store/orm_bindings/channel_node.py @@ -1,7 +1,6 @@ import random from datetime import datetime -from ipv8.database import database_blob from ipv8.keyvault.crypto import default_eccrypto from pony import orm @@ -65,7 +64,7 @@ class ChannelNode(db.Entity): reserved_flags = orm.Optional(int, size=16, default=0) origin_id = orm.Optional(int, size=64, default=0, index=True) - public_key = orm.Required(database_blob) + public_key = orm.Required(bytes) id_ = orm.Required(int, size=64) orm.composite_key(public_key, id_) orm.composite_index(public_key, origin_id) @@ -74,7 +73,7 @@ class ChannelNode(db.Entity): # Signature is nullable. This means that "None" entries are stored in DB as NULLs instead of empty strings. # NULLs are not checked for uniqueness and not indexed. # This is necessary to store unsigned signatures without violating the uniqueness constraints. - signature = orm.Optional(database_blob, unique=True, nullable=True, default=None) + signature = orm.Optional(bytes, unique=True, nullable=True, default=None) # Local added_on = orm.Optional(datetime, default=datetime.utcnow) @@ -108,7 +107,7 @@ def __init__(self, *args, **kwargs): # "sign_with" argument given, sign with it private_key_override = None if "sign_with" in kwargs: - kwargs["public_key"] = database_blob(kwargs["sign_with"].pub().key_to_bin()[10:]) + kwargs["public_key"] = kwargs["sign_with"].pub().key_to_bin()[10:] private_key_override = kwargs.pop("sign_with") # Free-for-all entries require special treatment @@ -140,8 +139,7 @@ def __init__(self, *args, **kwargs): if not private_key_override and not skip_key_check: # No key/signature given, sign with our own key. if ("signature" not in kwargs) and ( - ("public_key" not in kwargs) - or (kwargs["public_key"] == database_blob(self._my_key.pub().key_to_bin()[10:])) + ("public_key" not in kwargs) or (kwargs["public_key"] == self._my_key.pub().key_to_bin()[10:]) ): private_key_override = self._my_key @@ -222,7 +220,7 @@ def to_delete_file(self, filename): def sign(self, key=None): if not key: key = self._my_key - self.public_key = database_blob(key.pub().key_to_bin()[10:]) + self.public_key = key.pub().key_to_bin()[10:] _, self.signature = self._serialized(key) def has_valid_signature(self): @@ -251,7 +249,7 @@ def from_dict(cls, dct): @property @db_session def is_personal(self): - return database_blob(self._my_key.pub().key_to_bin()[10:]) == database_blob(self.public_key) + return self._my_key.pub().key_to_bin()[10:] == self.public_key @db_session def soft_delete(self): diff --git a/src/tribler-core/tribler_core/modules/metadata_store/orm_bindings/channel_peer.py b/src/tribler-core/tribler_core/modules/metadata_store/orm_bindings/channel_peer.py index d78a2e56500..11b23685531 100644 --- a/src/tribler-core/tribler_core/modules/metadata_store/orm_bindings/channel_peer.py +++ b/src/tribler-core/tribler_core/modules/metadata_store/orm_bindings/channel_peer.py @@ -1,7 +1,5 @@ from datetime import datetime -from ipv8.database import database_blob - from pony import orm @@ -13,7 +11,7 @@ class ChannelPeer(db.Entity): """ rowid = orm.PrimaryKey(int, size=64, auto=True) - public_key = orm.Required(database_blob, unique=True) + public_key = orm.Required(bytes, unique=True) individual_votes = orm.Set("ChannelVote", reverse='voter') added_on = orm.Optional(datetime, default=datetime.utcnow) diff --git a/src/tribler-core/tribler_core/modules/metadata_store/orm_bindings/collection_node.py b/src/tribler-core/tribler_core/modules/metadata_store/orm_bindings/collection_node.py index 6e896206627..f9d7f55da1f 100644 --- a/src/tribler-core/tribler_core/modules/metadata_store/orm_bindings/collection_node.py +++ b/src/tribler-core/tribler_core/modules/metadata_store/orm_bindings/collection_node.py @@ -1,8 +1,6 @@ import os from pathlib import Path -from ipv8.database import database_blob - from pony import orm from pony.orm import db_session, select @@ -22,7 +20,7 @@ UPDATED, ) from tribler_core.modules.metadata_store.orm_bindings.torrent_metadata import tdef_to_metadata_dict -from tribler_core.modules.metadata_store.serialization import COLLECTION_NODE, CollectionNodePayload, CHANNEL_TORRENT +from tribler_core.modules.metadata_store.serialization import CHANNEL_TORRENT, COLLECTION_NODE, CollectionNodePayload from tribler_core.utilities.random_utils import random_infohash # pylint: disable=too-many-statements @@ -57,8 +55,10 @@ def state(self): return CHANNEL_STATE.PERSONAL.value toplevel_parent = self.get_parent_nodes()[0] - if (toplevel_parent.metadata_type == CHANNEL_TORRENT and - toplevel_parent.local_version == toplevel_parent.timestamp): + if ( + toplevel_parent.metadata_type == CHANNEL_TORRENT + and toplevel_parent.local_version == toplevel_parent.timestamp + ): return CHANNEL_STATE.COMPLETE.value return CHANNEL_STATE.PREVIEW.value @@ -94,7 +94,7 @@ def copy_torrent_from_infohash(self, infohash): :return: New TorrentMetadata signed with your key. """ - existing = db.TorrentMetadata.select(lambda g: g.infohash == database_blob(infohash)).first() + existing = db.TorrentMetadata.select(lambda g: g.infohash == infohash).first() if not existing: return None @@ -298,8 +298,7 @@ def update_node_info(n): dead_parents.remove(0) # Delete orphans db.ChannelNode.select( - lambda g: database_blob(db.ChannelNode._my_key.pub().key_to_bin()[10:]) # pylint: disable=W0212 - == g.public_key + lambda g: db.ChannelNode._my_key.pub().key_to_bin()[10:] == g.public_key # pylint: disable=W0212 and g.origin_id in dead_parents ).delete() orm.flush() # Just in case... diff --git a/src/tribler-core/tribler_core/modules/metadata_store/orm_bindings/torrent_metadata.py b/src/tribler-core/tribler_core/modules/metadata_store/orm_bindings/torrent_metadata.py index 997b50154c6..c25e0ad3ff8 100644 --- a/src/tribler-core/tribler_core/modules/metadata_store/orm_bindings/torrent_metadata.py +++ b/src/tribler-core/tribler_core/modules/metadata_store/orm_bindings/torrent_metadata.py @@ -1,8 +1,6 @@ from datetime import datetime from struct import unpack -from ipv8.database import database_blob - from pony import orm from pony.orm import db_session @@ -55,7 +53,7 @@ class TorrentMetadata(db.MetadataNode): _discriminator_ = REGULAR_TORRENT # Serializable - infohash = orm.Required(database_blob, index=True) + infohash = orm.Required(bytes, index=True) size = orm.Optional(int, size=64, default=0) torrent_date = orm.Optional(datetime, default=datetime.utcnow, index=True) tracker_info = orm.Optional(str, default='') @@ -112,8 +110,8 @@ def add_ffa_from_dict(cls, ffa_dict): # Check that this torrent is yet unknown to GigaChannel, and if there is no duplicate FFA entry. # Test for a duplicate id_+public_key is necessary to account for a (highly improbable) situation when # two entries have different infohashes but the same id_. We do not want people to exploit this. - ih_blob = database_blob(ffa_dict["infohash"]) - pk_blob = database_blob(b"") + ih_blob = ffa_dict["infohash"] + pk_blob = b"" if cls.exists(lambda g: (g.infohash == ih_blob) or (g.id_ == id_ and g.public_key == pk_blob)): return None # Add the torrent as a free-for-all entry if it is unknown to GigaChannel @@ -152,7 +150,7 @@ def metadata_conflicting(self, b): @classmethod @db_session def get_with_infohash(cls, infohash): - return cls.select(lambda g: g.infohash == database_blob(infohash)).first() + return cls.select(lambda g: g.infohash == infohash).first() @classmethod @db_session diff --git a/src/tribler-core/tribler_core/modules/metadata_store/orm_bindings/torrent_state.py b/src/tribler-core/tribler_core/modules/metadata_store/orm_bindings/torrent_state.py index 35e77d3b4fd..0e501100226 100644 --- a/src/tribler-core/tribler_core/modules/metadata_store/orm_bindings/torrent_state.py +++ b/src/tribler-core/tribler_core/modules/metadata_store/orm_bindings/torrent_state.py @@ -1,5 +1,3 @@ -from ipv8.database import database_blob - from pony import orm @@ -10,7 +8,7 @@ class TorrentState(db.Entity): """ rowid = orm.PrimaryKey(int, auto=True) - infohash = orm.Required(database_blob, unique=True) + infohash = orm.Required(bytes, unique=True) seeders = orm.Optional(int, default=0) leechers = orm.Optional(int, default=0) last_check = orm.Optional(int, size=64, default=0) diff --git a/src/tribler-core/tribler_core/modules/metadata_store/payload_checker.py b/src/tribler-core/tribler_core/modules/metadata_store/payload_checker.py index c7640eeb606..141ff9189c4 100644 --- a/src/tribler-core/tribler_core/modules/metadata_store/payload_checker.py +++ b/src/tribler-core/tribler_core/modules/metadata_store/payload_checker.py @@ -1,8 +1,6 @@ import enum from dataclasses import dataclass, field -from ipv8.database import database_blob - from pony.orm import db_session from tribler_core.modules.category_filter.l2_filter import is_forbidden @@ -187,9 +185,7 @@ def update_local_node(self): If we don't have some version of the node locally, CONTINUE control to further checks. """ # Check for the older version of the added node - node = self.mds.ChannelNode.get_for_update( - public_key=database_blob(self.payload.public_key), id_=self.payload.id_ - ) + node = self.mds.ChannelNode.get_for_update(public_key=self.payload.public_key, id_=self.payload.id_) if not node: return CONTINUE diff --git a/src/tribler-core/tribler_core/modules/metadata_store/restapi/channels_endpoint.py b/src/tribler-core/tribler_core/modules/metadata_store/restapi/channels_endpoint.py index 10a7348da88..12ff8d42439 100644 --- a/src/tribler-core/tribler_core/modules/metadata_store/restapi/channels_endpoint.py +++ b/src/tribler-core/tribler_core/modules/metadata_store/restapi/channels_endpoint.py @@ -10,7 +10,6 @@ from aiohttp_apispec import docs, json_schema from ipv8.REST.schema import schema -from ipv8.database import database_blob from marshmallow.fields import Boolean, Dict, Integer, String @@ -260,9 +259,7 @@ async def copy_channel(self, request): channel_pk, channel_id = self.get_channel_from_request(request) personal_root = channel_id == 0 and channel_pk == self.session.mds.my_key.pub().key_to_bin()[10:] # TODO: better error handling - target_collection = self.session.mds.CollectionNode.get( - public_key=database_blob(channel_pk), id_=channel_id - ) + target_collection = self.session.mds.CollectionNode.get(public_key=channel_pk, id_=channel_id) try: request_parsed = await request.json() except (ContentTypeError, ValueError): @@ -272,7 +269,7 @@ async def copy_channel(self, request): return RESTResponse({"error": "Target channel not found"}, status=HTTP_NOT_FOUND) results_list = [] for entry in request_parsed: - public_key, id_ = database_blob(unhexlify(entry["public_key"])), entry["id"] + public_key, id_ = unhexlify(entry["public_key"]), entry["id"] source = self.session.mds.ChannelNode.get(public_key=public_key, id_=id_) if not source: return RESTResponse({"error": "Source entry not found"}, status=HTTP_BAD_REQUEST) @@ -358,7 +355,7 @@ async def create_collection(self, request): async def add_torrent_to_channel(self, request): channel_pk, channel_id = self.get_channel_from_request(request) with db_session: - channel = self.session.mds.CollectionNode.get(public_key=database_blob(channel_pk), id_=channel_id) + channel = self.session.mds.CollectionNode.get(public_key=channel_pk, id_=channel_id) if not channel: return RESTResponse({"error": "Unknown channel"}, status=HTTP_NOT_FOUND) @@ -442,7 +439,7 @@ async def post_commit(self, request): for t in self.session.mds.CollectionNode.commit_all_channels(): self.session.gigachannel_manager.updated_my_channel(TorrentDef.load_from_dict(t)) else: - coll = self.session.mds.CollectionNode.get(public_key=database_blob(channel_pk), id_=channel_id) + coll = self.session.mds.CollectionNode.get(public_key=channel_pk, id_=channel_id) if not coll: return RESTResponse({"success": False}, status=HTTP_NOT_FOUND) torrent_dict = coll.commit_channel_torrent() @@ -460,7 +457,7 @@ async def is_channel_dirty(self, request): channel_pk, _ = self.get_channel_from_request(request) with db_session: dirty = self.session.mds.MetadataNode.exists( - lambda g: g.public_key == database_blob(channel_pk) and g.status in DIRTY_STATUSES + lambda g: g.public_key == channel_pk and g.status in DIRTY_STATUSES ) return RESTResponse({"dirty": dirty}) diff --git a/src/tribler-core/tribler_core/modules/metadata_store/restapi/metadata_endpoint.py b/src/tribler-core/tribler_core/modules/metadata_store/restapi/metadata_endpoint.py index bd7bc4c920a..506a8fb827e 100644 --- a/src/tribler-core/tribler_core/modules/metadata_store/restapi/metadata_endpoint.py +++ b/src/tribler-core/tribler_core/modules/metadata_store/restapi/metadata_endpoint.py @@ -5,7 +5,6 @@ from aiohttp_apispec import docs from ipv8.REST.schema import schema -from ipv8.database import database_blob from marshmallow.fields import Integer, String @@ -88,7 +87,7 @@ async def update_channel_entries(self, request): return RESTResponse({"error": "Bad JSON"}, status=HTTP_BAD_REQUEST) results_list = [] for entry in request_parsed: - public_key = database_blob(unhexlify(entry.pop("public_key"))) + public_key = unhexlify(entry.pop("public_key")) id_ = entry.pop("id") error, result = self.update_entry(public_key, id_, entry) # TODO: handle the results for a list that contains some errors in a smarter way @@ -119,7 +118,7 @@ async def delete_channel_entries(self, request): request_parsed = await request.json() results_list = [] for entry in request_parsed: - public_key = database_blob(unhexlify(entry.pop("public_key"))) + public_key = unhexlify(entry.pop("public_key")) id_ = entry.pop("id") entry = self.session.mds.ChannelNode.get(public_key=public_key, id_=id_) if not entry: @@ -159,7 +158,7 @@ async def get_channel_entries(self, request): public_key = unhexlify(request.match_info['public_key']) id_ = request.match_info['id'] with db_session: - entry = self.session.mds.ChannelNode.get(public_key=database_blob(public_key), id_=id_) + entry = self.session.mds.ChannelNode.get(public_key=public_key, id_=id_) if entry: # TODO: handle costly attributes in a more graceful and generic way for all types of metadata diff --git a/src/tribler-core/tribler_core/modules/metadata_store/serialization.py b/src/tribler-core/tribler_core/modules/metadata_store/serialization.py index 96a422bffa0..126cf366738 100644 --- a/src/tribler-core/tribler_core/modules/metadata_store/serialization.py +++ b/src/tribler-core/tribler_core/modules/metadata_store/serialization.py @@ -4,7 +4,6 @@ from datetime import datetime, timedelta from typing import List, Tuple -from ipv8.database import database_blob from ipv8.keyvault.crypto import default_eccrypto from ipv8.messaging.lazy_payload import VariablePayload, vp_compile from ipv8.messaging.payload import Payload @@ -66,7 +65,7 @@ class UnknownBlobTypeException(Exception): def read_payload_with_offset(data, offset=0): # First we have to determine the actual payload type - metadata_type = struct.unpack_from('>H', database_blob(data), offset=offset)[0] + metadata_type = struct.unpack_from('>H', data, offset=offset)[0] payload_class = DISCRIMINATOR_TO_PAYLOAD_CLASS.get(metadata_type) if payload_class is not None: return payload_class.from_signed_blob_with_offset(data, offset=offset) diff --git a/src/tribler-core/tribler_core/modules/metadata_store/store.py b/src/tribler-core/tribler_core/modules/metadata_store/store.py index 2ad6f719615..55317956e6e 100644 --- a/src/tribler-core/tribler_core/modules/metadata_store/store.py +++ b/src/tribler-core/tribler_core/modules/metadata_store/store.py @@ -4,8 +4,6 @@ from datetime import datetime, timedelta from time import sleep, time -from ipv8.database import database_blob - from lz4.frame import LZ4FrameDecompressor from pony import orm @@ -148,7 +146,7 @@ def __init__( self.db_filename = db_filename self.channels_dir = channels_dir self.my_key = my_key - self.my_public_key_bin = bytes(database_blob(self.my_key.pub().key_to_bin()[10:])) + self.my_public_key_bin = self.my_key.pub().key_to_bin()[10:] self._logger = logging.getLogger(self.__class__.__name__) self._shutting_down = False @@ -565,9 +563,7 @@ def torrent_exists_in_personal_channel(self, infohash): :return: True if torrent exists else False """ return self.TorrentMetadata.exists( - lambda g: g.public_key == self.my_public_key_bin - and g.infohash == database_blob(infohash) - and g.status != LEGACY_ENTRY + lambda g: g.public_key == self.my_public_key_bin and g.infohash == infohash and g.status != LEGACY_ENTRY ) # pylint: disable=unused-argument diff --git a/src/tribler-core/tribler_core/modules/metadata_store/tests/test_channel_download.py b/src/tribler-core/tribler_core/modules/metadata_store/tests/test_channel_download.py index 303223285bd..cd9d7bb86d1 100644 --- a/src/tribler-core/tribler_core/modules/metadata_store/tests/test_channel_download.py +++ b/src/tribler-core/tribler_core/modules/metadata_store/tests/test_channel_download.py @@ -1,4 +1,3 @@ -from ipv8.database import database_blob from ipv8.util import succeed from pony.orm import db_session @@ -56,7 +55,7 @@ def hinted_start_download(tdef=None, config=None, hidden=False): with db_session: # There should be 8 torrents + 1 channel torrent - channel2 = session.mds.ChannelMetadata.get(public_key=database_blob(payload.public_key)) + channel2 = session.mds.ChannelMetadata.get(public_key=payload.public_key) assert channel2.timestamp == channel2.local_version assert channel2.timestamp == 1565621688018 assert session.mds.ChannelNode.select().count() == 8 diff --git a/src/tribler-core/tribler_core/modules/metadata_store/tests/test_channel_metadata.py b/src/tribler-core/tribler_core/modules/metadata_store/tests/test_channel_metadata.py index f849a978bb9..dc1b4bb798e 100644 --- a/src/tribler-core/tribler_core/modules/metadata_store/tests/test_channel_metadata.py +++ b/src/tribler-core/tribler_core/modules/metadata_store/tests/test_channel_metadata.py @@ -6,7 +6,6 @@ from time import sleep from unittest.mock import Mock, patch -from ipv8.database import database_blob from ipv8.keyvault.crypto import default_eccrypto from lz4.frame import LZ4FrameDecompressor @@ -28,12 +27,11 @@ from tribler_core.modules.metadata_store.serialization import CHANNEL_TORRENT, COLLECTION_NODE, REGULAR_TORRENT from tribler_core.modules.metadata_store.store import HealthItemsPayload from tribler_core.tests.tools.common import TESTS_DATA_DIR, TORRENT_UBUNTU_FILE -from tribler_core.utilities import path_util from tribler_core.utilities.random_utils import random_infohash - # pylint: disable=protected-access + @pytest.fixture def my_key(): return default_eccrypto.generate_key("curve25519") @@ -47,12 +45,12 @@ def torrent_template(): @pytest.fixture def sample_torrent_dict(my_key): return { - "infohash": database_blob(b"1" * 20), + "infohash": b"1" * 20, "size": 123, "torrent_date": datetime.utcnow(), "tags": "bla", "id_": 123, - "public_key": database_blob(my_key.pub().key_to_bin()[10:]), + "public_key": my_key.pub().key_to_bin()[10:], "title": "lalala", } @@ -171,7 +169,7 @@ def test_get_channel_with_dirname(sample_channel_dict, metadata_store): assert channel_metadata == channel_result # Test for corner-case of channel PK starting with zeroes - channel_metadata.public_key = database_blob(unhexlify('0' * 128)) + channel_metadata.public_key = unhexlify('0' * 128) channel_result = metadata_store.ChannelMetadata.get_channel_with_dirname(channel_metadata.dirname) assert channel_metadata == channel_result @@ -831,7 +829,7 @@ def test_get_channel_name(metadata_store): """ infohash = b"\x00" * 20 title = "testchan" - chan = metadata_store.ChannelMetadata(title=title, infohash=database_blob(infohash)) + chan = metadata_store.ChannelMetadata(title=title, infohash=infohash) dirname = chan.dirname assert title == metadata_store.ChannelMetadata.get_channel_name(dirname, infohash) diff --git a/src/tribler-core/tribler_core/modules/metadata_store/tests/test_gigachannel_manager.py b/src/tribler-core/tribler_core/modules/metadata_store/tests/test_gigachannel_manager.py index 237ccce7a23..8ad30eb579f 100644 --- a/src/tribler-core/tribler_core/modules/metadata_store/tests/test_gigachannel_manager.py +++ b/src/tribler-core/tribler_core/modules/metadata_store/tests/test_gigachannel_manager.py @@ -4,7 +4,6 @@ from pathlib import Path from unittest.mock import Mock, patch -from ipv8.database import database_blob from ipv8.util import succeed from pony.orm import db_session @@ -12,8 +11,8 @@ import pytest from tribler_common.simpledefs import DLSTATUS_SEEDING -from tribler_core.config.tribler_config import TriblerConfig +from tribler_core.config.tribler_config import TriblerConfig from tribler_core.modules.libtorrent.torrentdef import TorrentDef from tribler_core.modules.metadata_store.gigachannel_manager import GigaChannelManager from tribler_core.modules.metadata_store.orm_bindings.channel_node import NEW @@ -161,8 +160,8 @@ async def test_check_channels_updates(enable_chant, personal_channel, channel_ma # Subscribed, not updated session.mds.ChannelMetadata( title="bla1", - public_key=database_blob(b'123'), - signature=database_blob(b'345'), + public_key=b'123', + signature=b'345', skip_key_check=True, timestamp=123, local_version=123, @@ -172,8 +171,8 @@ async def test_check_channels_updates(enable_chant, personal_channel, channel_ma # Not subscribed, updated session.mds.ChannelMetadata( title="bla2", - public_key=database_blob(b'124'), - signature=database_blob(b'346'), + public_key=b'124', + signature=b'346', skip_key_check=True, timestamp=123, local_version=122, @@ -183,8 +182,8 @@ async def test_check_channels_updates(enable_chant, personal_channel, channel_ma # Subscribed, updated - only this one should be downloaded chan3 = session.mds.ChannelMetadata( title="bla3", - public_key=database_blob(b'125'), - signature=database_blob(b'347'), + public_key=b'125', + signature=b'347', skip_key_check=True, timestamp=123, local_version=122, @@ -201,7 +200,7 @@ def mock_download_channel(chan1): @db_session def fake_get_metainfo(infohash, **_): - return {'info': {'name': session.mds.ChannelMetadata.get(infohash=database_blob(infohash)).dirname}} + return {'info': {'name': session.mds.ChannelMetadata.get(infohash=infohash).dirname}} session.dlmgr.get_metainfo = fake_get_metainfo session.dlmgr.metainfo_requests = {} @@ -251,9 +250,9 @@ async def test_remove_cruft_channels( # Now we add an external channel we are subscribed to. chan2 = session.mds.ChannelMetadata( title="bla1", - infohash=database_blob(b'123'), - public_key=database_blob(b'123'), - signature=database_blob(b'345'), + infohash=b'123', + public_key=b'123', + signature=b'345', skip_key_check=True, timestamp=123, local_version=123, @@ -263,9 +262,9 @@ async def test_remove_cruft_channels( # Another external channel, but there is a catch: we recently unsubscribed from it chan3 = session.mds.ChannelMetadata( title="bla2", - infohash=database_blob(b'124'), - public_key=database_blob(b'124'), - signature=database_blob(b'346'), + infohash=b'124', + public_key=b'124', + signature=b'346', skip_key_check=True, timestamp=123, local_version=123, @@ -290,16 +289,16 @@ def get_def(self): # Double conversion is required to make sure that buffers signatures are not the same mock_dl_list = [ # Downloads for the personal channel - MockDownload(database_blob(bytes(my_chan_old_infohash)), personal_channel.dirname), - MockDownload(database_blob(bytes(personal_channel.infohash)), personal_channel.dirname), + MockDownload(my_chan_old_infohash, personal_channel.dirname), + MockDownload(personal_channel.infohash, personal_channel.dirname), # Downloads for the updated external channel: "old ones" and "recent" - MockDownload(database_blob(b'12331244'), chan2.dirname), - MockDownload(database_blob(bytes(chan2.infohash)), chan2.dirname), + MockDownload(b'12331244', chan2.dirname), + MockDownload(chan2.infohash, chan2.dirname), # Downloads for the unsubscribed external channel - MockDownload(database_blob(b'1231551'), chan3.dirname), - MockDownload(database_blob(bytes(chan3.infohash)), chan3.dirname), + MockDownload(b'1231551', chan3.dirname), + MockDownload(chan3.infohash, chan3.dirname), # Orphaned download - MockDownload(database_blob(b'333'), "blabla"), + MockDownload(b'333', "blabla"), ] def mock_get_channel_downloads(**_): @@ -331,12 +330,12 @@ def mock_remove(infohash, remove_content=False): @pytest.mark.asyncio -async def test_reject_malformed_channel(enable_chant, channel_manager, mock_dlmgr, session, tmpdir): # pylint: disable=unused-argument, redefined-outer-name +async def test_reject_malformed_channel( + enable_chant, channel_manager, mock_dlmgr, session, tmpdir +): # pylint: disable=unused-argument, redefined-outer-name global initiated_download with db_session: - channel = session.mds.ChannelMetadata( - title="bla1", public_key=database_blob(b'123'), infohash=random_infohash() - ) + channel = session.mds.ChannelMetadata(title="bla1", public_key=b'123', infohash=random_infohash()) session.config = TriblerConfig(state_dir=tmpdir) def mock_get_metainfo_bad(*args, **kwargs): diff --git a/src/tribler-core/tribler_core/modules/metadata_store/tests/test_metadata.py b/src/tribler-core/tribler_core/modules/metadata_store/tests/test_metadata.py index 8473783a1f5..4fb6ccbbf85 100644 --- a/src/tribler-core/tribler_core/modules/metadata_store/tests/test_metadata.py +++ b/src/tribler-core/tribler_core/modules/metadata_store/tests/test_metadata.py @@ -1,4 +1,3 @@ -from ipv8.database import database_blob from ipv8.keyvault.crypto import default_eccrypto from pony import orm @@ -122,7 +121,7 @@ def test_has_valid_signature(metadata_store): # Create metadata with wrong key metadata.delete() - md_dict.update(public_key=database_blob(b"aaa")) + md_dict.update(public_key=b"aaa") md_dict.pop("rowid") metadata = metadata_store.ChannelNode(skip_key_check=True, **md_dict) @@ -130,7 +129,7 @@ def test_has_valid_signature(metadata_store): key = default_eccrypto.generate_key("curve25519") metadata2 = metadata_store.ChannelNode(sign_with=key, **md_dict) - assert database_blob(key.pub().key_to_bin()[10:]), metadata2.public_key + assert key.pub().key_to_bin()[10:], metadata2.public_key md_dict2 = metadata2.to_dict() md_dict2["signature"] = md_dict["signature"] with pytest.raises(InvalidSignatureException): diff --git a/src/tribler-core/tribler_core/modules/metadata_store/tests/test_store.py b/src/tribler-core/tribler_core/modules/metadata_store/tests/test_store.py index e055958109a..b00b85be090 100644 --- a/src/tribler-core/tribler_core/modules/metadata_store/tests/test_store.py +++ b/src/tribler-core/tribler_core/modules/metadata_store/tests/test_store.py @@ -6,7 +6,6 @@ from datetime import datetime from unittest.mock import patch -from ipv8.database import database_blob from ipv8.keyvault.crypto import default_eccrypto from pony.orm import db_session @@ -35,7 +34,7 @@ def get_payloads(entity_class, key): orig_key = entity_class._my_key entity_class._my_key = key - c = entity_class(infohash=database_blob(random_infohash())) + c = entity_class(infohash=random_infohash()) payload = c._payload_class.from_signed_blob(c.serialized()) deleted_payload = DeletedMetadataPayload.from_signed_blob(c.serialized_delete()) @@ -45,9 +44,7 @@ def get_payloads(entity_class, key): def make_wrong_payload(filename): key = default_eccrypto.generate_key("curve25519") - metadata_payload = SignedPayload( - 666, 0, database_blob(key.pub().key_to_bin()[10:]), signature=b'\x00' * 64, skip_key_check=True - ) + metadata_payload = SignedPayload(666, 0, key.pub().key_to_bin()[10:], signature=b'\x00' * 64, skip_key_check=True) with open(filename, 'wb') as output_file: output_file.write(metadata_payload.serialized()) @@ -67,7 +64,7 @@ def test_process_channel_dir_file(tmpdir, metadata_store): """ Test whether we are able to process files in a directory containing node metadata """ - test_node_metadata = metadata_store.TorrentMetadata(title='test', infohash=database_blob(random_infohash())) + test_node_metadata = metadata_store.TorrentMetadata(title='test', infohash=random_infohash()) metadata_path = tmpdir / 'metadata.data' test_node_metadata.to_file(metadata_path) # We delete this TorrentMeta info now, it should be added again to the database when loading it @@ -101,7 +98,7 @@ def test_squash_mdblobs(metadata_store): md_list = [ metadata_store.TorrentMetadata( title=''.join(r.choice(string.ascii_uppercase + string.digits) for _ in range(20)), - infohash=database_blob(random_infohash()), + infohash=random_infohash(), torrent_date=datetime.utcfromtimestamp(100), ) for _ in range(0, 10) @@ -122,7 +119,7 @@ def test_squash_mdblobs_multiple_chunks(metadata_store): md_list = [ metadata_store.TorrentMetadata( title=''.join(r.choice(string.ascii_uppercase + string.digits) for _ in range(20)), - infohash=database_blob(random_infohash()), + infohash=random_infohash(), torrent_date=datetime.utcfromtimestamp(100), ) for _ in range(0, 10) @@ -155,7 +152,7 @@ def test_multiple_squashed_commit_and_read(metadata_store): channel = metadata_store.ChannelMetadata.create_channel('testchan') md_list = [ metadata_store.TorrentMetadata( - origin_id=channel.id_, title='test' + str(x), status=NEW, infohash=database_blob(random_infohash()) + origin_id=channel.id_, title='test' + str(x), status=NEW, infohash=random_infohash() ) for x in range(0, num_entries) ] @@ -180,7 +177,7 @@ def test_skip_processing_of_received_personal_channel_torrents(metadata_store): """ channel = metadata_store.ChannelMetadata.create_channel('testchan') torrent_md = metadata_store.TorrentMetadata( - origin_id=channel.id_, title='test', status=NEW, infohash=database_blob(random_infohash()) + origin_id=channel.id_, title='test', status=NEW, infohash=random_infohash() ) channel.commit_channel_torrent() torrent_md.delete() @@ -208,7 +205,7 @@ def test_skip_processing_mdblob_with_forbidden_terms(metadata_store): Test that an mdblob with forbidden terms cannot ever get into the local database """ key = default_eccrypto.generate_key("curve25519") - chan_entry = metadata_store.ChannelMetadata(title="12yo", infohash=database_blob(random_infohash()), sign_with=key) + chan_entry = metadata_store.ChannelMetadata(title="12yo", infohash=random_infohash(), sign_with=key) chan_payload = chan_entry._payload_class(**chan_entry.to_dict()) chan_entry.delete() assert metadata_store.process_payload(chan_payload) == [] @@ -321,7 +318,7 @@ def test_process_payload_with_known_channel_public_key(metadata_store): """ key1 = default_eccrypto.generate_key("curve25519") key2 = default_eccrypto.generate_key("curve25519") - torrent = metadata_store.TorrentMetadata(infohash=database_blob(random_infohash()), sign_with=key1) + torrent = metadata_store.TorrentMetadata(infohash=random_infohash(), sign_with=key1) payload = torrent._payload_class(**torrent.to_dict()) torrent.delete() # Check rejecting a payload with non-matching public key @@ -346,11 +343,11 @@ def test_process_payload_reject_older(metadata_store): version=123, timestamp=12, local_version=12, - infohash=database_blob(random_infohash()), + infohash=random_infohash(), sign_with=key, ) torrent = metadata_store.TorrentMetadata( - title='blabla', timestamp=11, origin_id=channel.id_, infohash=database_blob(random_infohash()), sign_with=key + title='blabla', timestamp=11, origin_id=channel.id_, infohash=random_infohash(), sign_with=key ) payload = torrent._payload_class(**torrent.to_dict()) torrent.delete() @@ -362,7 +359,7 @@ def test_process_payload_reject_older(metadata_store): folder_2 = metadata_store.CollectionNode(origin_id=folder_1.id_, sign_with=key) torrent = metadata_store.TorrentMetadata( - title='blabla', timestamp=11, origin_id=folder_2.id_, infohash=database_blob(random_infohash()), sign_with=key + title='blabla', timestamp=11, origin_id=folder_2.id_, infohash=random_infohash(), sign_with=key ) payload = torrent._payload_class(**torrent.to_dict()) torrent.delete() @@ -373,7 +370,7 @@ def test_process_payload_reject_older(metadata_store): folder_1 = metadata_store.CollectionNode(origin_id=123123, sign_with=key) folder_2 = metadata_store.CollectionNode(origin_id=folder_1.id_, sign_with=key) torrent = metadata_store.TorrentMetadata( - title='blabla', timestamp=11, origin_id=folder_2.id_, infohash=database_blob(random_infohash()), sign_with=key + title='blabla', timestamp=11, origin_id=folder_2.id_, infohash=random_infohash(), sign_with=key ) payload = torrent._payload_class(**torrent.to_dict()) torrent.delete() @@ -389,13 +386,13 @@ def test_process_payload_reject_older_entry(metadata_store): """ key = default_eccrypto.generate_key("curve25519") torrent_old = metadata_store.TorrentMetadata( - title='blabla', timestamp=11, id_=3, infohash=database_blob(random_infohash()), sign_with=key + title='blabla', timestamp=11, id_=3, infohash=random_infohash(), sign_with=key ) payload_old = torrent_old._payload_class(**torrent_old.to_dict()) torrent_old.delete() torrent_updated = metadata_store.TorrentMetadata( - title='blabla', timestamp=12, id_=3, infohash=database_blob(random_infohash()), sign_with=key + title='blabla', timestamp=12, id_=3, infohash=random_infohash(), sign_with=key ) # Test rejecting older version of the same entry assert metadata_store.process_payload(payload_old, skip_personal_metadata_payload=False)[0] == ProcessingResult( @@ -405,15 +402,15 @@ def test_process_payload_reject_older_entry(metadata_store): @db_session def test_get_num_channels_nodes(metadata_store): - metadata_store.ChannelMetadata(title='testchan', id_=0, infohash=database_blob(random_infohash())) - metadata_store.ChannelMetadata(title='testchan', id_=123, infohash=database_blob(random_infohash())) + metadata_store.ChannelMetadata(title='testchan', id_=0, infohash=random_infohash()) + metadata_store.ChannelMetadata(title='testchan', id_=123, infohash=random_infohash()) metadata_store.ChannelMetadata( title='testchan', id_=0, public_key=unhexlify('0' * 20), signature=unhexlify('0' * 64), skip_key_check=True, - infohash=database_blob(random_infohash()), + infohash=random_infohash(), ) metadata_store.ChannelMetadata( title='testchan', @@ -421,11 +418,11 @@ def test_get_num_channels_nodes(metadata_store): public_key=unhexlify('1' * 20), signature=unhexlify('1' * 64), skip_key_check=True, - infohash=database_blob(random_infohash()), + infohash=random_infohash(), ) _ = [ - metadata_store.TorrentMetadata(title='test' + str(x), status=NEW, infohash=database_blob(random_infohash())) + metadata_store.TorrentMetadata(title='test' + str(x), status=NEW, infohash=random_infohash()) for x in range(0, 3) ] diff --git a/src/tribler-core/tribler_core/modules/metadata_store/tests/test_torrent_metadata.py b/src/tribler-core/tribler_core/modules/metadata_store/tests/test_torrent_metadata.py index da986945dbd..8a4ec467bb1 100644 --- a/src/tribler-core/tribler_core/modules/metadata_store/tests/test_torrent_metadata.py +++ b/src/tribler-core/tribler_core/modules/metadata_store/tests/test_torrent_metadata.py @@ -1,7 +1,6 @@ from datetime import datetime from time import time -from ipv8.database import database_blob from ipv8.keyvault.crypto import default_eccrypto from pony import orm @@ -17,7 +16,7 @@ from tribler_core.tests.tools.common import TORRENT_UBUNTU_FILE from tribler_core.utilities.random_utils import random_infohash -EMPTY_BLOB = database_blob(b"") +EMPTY_BLOB = b"" def rnd_torrent(): diff --git a/src/tribler-core/tribler_core/modules/torrent_checker/torrent_checker.py b/src/tribler-core/tribler_core/modules/torrent_checker/torrent_checker.py index 71bd453ce28..8f70146f09f 100644 --- a/src/tribler-core/tribler_core/modules/torrent_checker/torrent_checker.py +++ b/src/tribler-core/tribler_core/modules/torrent_checker/torrent_checker.py @@ -4,7 +4,6 @@ import time from asyncio import CancelledError, gather -from ipv8.database import database_blob from ipv8.taskmanager import TaskManager, task from pony.orm import db_session, desc, select @@ -232,7 +231,7 @@ def is_blacklisted_tracker(self, tracker_url): @db_session def get_valid_trackers_of_torrent(self, torrent_id): """ Get a set of valid trackers for torrent. Also remove any invalid torrent.""" - db_tracker_list = self.tribler_session.mds.TorrentState.get(infohash=database_blob(torrent_id)).trackers + db_tracker_list = self.tribler_session.mds.TorrentState.get(infohash=torrent_id).trackers return {tracker.url for tracker in db_tracker_list if is_valid_url(tracker.url) and not self.is_blacklisted_tracker(tracker.url)} @@ -303,7 +302,7 @@ async def check_torrent_health(self, infohash, timeout=20, scrape_now=False): # We first check whether the torrent is already in the database and checked before with db_session: - result = self.tribler_session.mds.TorrentState.get(infohash=database_blob(infohash)) + result = self.tribler_session.mds.TorrentState.get(infohash=infohash) if result: torrent_id = result.infohash last_check = result.last_check @@ -382,7 +381,7 @@ def _update_torrent_result(self, response): with db_session: # Update torrent state - torrent = self.tribler_session.mds.TorrentState.get(infohash=database_blob(infohash)) + torrent = self.tribler_session.mds.TorrentState.get(infohash=infohash) if not torrent: self._logger.warning( "Tried to update torrent health data in DB for an unknown torrent: %s", hexlify(infohash))