Various unit test fixes

src/tribler-core/tribler_core/modules/libtorrent/tests/test_torrent_def.py

@@ -12,7 +12,7 @@
 from tribler_core.tests.tools.common import TESTS_DATA_DIR, TORRENT_UBUNTU_FILE
 from tribler_core.tests.tools.test_as_server import BaseTestCase
 from tribler_core.tests.tools.tools import timeout
-from tribler_core.utilities.path_util import mkdtemp
+from tribler_core.utilities.path_util import Path, mkdtemp
 from tribler_core.utilities.utilities import bdecode_compat
 
 TRACKER = 'http://www.tribler.org/announce'
@@ -278,3 +278,26 @@ def test_get_name_as_unicode(self):
         self.assertEqual(t.get_name_as_unicode(), name_unicode)
         t.metainfo = {b'info': {b'name': b'test\xff' + name_bytes}}
         self.assertEqual(t.get_name_as_unicode(), 'test?????????????')
+
+    def test_get_files_with_length(self):
+        name_bytes = b'\xe8\xaf\xad\xe8\xa8\x80\xe5\xa4\x84\xe7\x90\x86'
+        name_unicode = name_bytes.decode()
+        t = TorrentDef()
+        t.metainfo = {b'info': {b'files': [{b'path.utf-8': [name_bytes], b'length': 123},
+                                           {b'path.utf-8': [b'file.txt'], b'length': 456}]}}
+        self.assertEqual(t.get_files_with_length(), [(Path(name_unicode), 123),
+                                                     (Path('file.txt'), 456)])
+
+        t.metainfo = {b'info': {b'files': [{b'path': [name_bytes], b'length': 123},
+                                           {b'path': [b'file.txt'], b'length': 456}]}}
+        self.assertEqual(t.get_files_with_length(), [(Path(name_unicode), 123),
+                                                     (Path('file.txt'), 456)])
+
+        t.metainfo = {b'info': {b'files': [{b'path': [b'test\xff' + name_bytes], b'length': 123},
+                                           {b'path': [b'file.txt'], b'length': 456}]}}
+        self.assertEqual(t.get_files_with_length(), [(Path('test?????????????'), 123),
+                                                     (Path('file.txt'), 456)])
+
+        t.metainfo = {b'info': {b'files': [{b'path.utf-8': [b'test\xff' + name_bytes], b'length': 123},
+                                           {b'path': [b'file.txt'], b'length': 456}]}}
+        self.assertEqual(t.get_files_with_length(), [(Path('file.txt'), 456)])

src/tribler-core/tribler_core/modules/libtorrent/torrentdef.py

@@ -311,10 +311,10 @@ def filter_characters(name):
                     def filter_character(char):
                         if 0 < char < 128:
                             return chr(char)
-                        self._logger.debug("Bad character %s", bytes(char))
-                        return u"?"
-                    return u"".join([filter_character(char) for char in name])
-                return str(filter_characters(self.metainfo[b"info"][b"name"]))
+                        self._logger.debug("Bad character 0x%X", char)
+                        return "?"
+                    return "".join([filter_character(char) for char in name])
+                return filter_characters(self.metainfo[b"info"][b"name"])
             except UnicodeError:
                 pass
 
@@ -381,28 +381,18 @@ def _get_all_files_as_unicode_with_length(self):
                     except UnicodeError:
                         pass
 
-                    # Try to convert the names in path to unicode,
-                    # without specifying the encoding
-                    try:
-                        yield join(*[str(element) for element in file_dict[b"path"]]), file_dict[b"length"]
-                        continue
-                    except UnicodeError:
-                        pass
-
                     # Convert the names in path to unicode by
                     # replacing out all characters that may -even
                     # remotely- cause problems with the '?' character
                     try:
                         def filter_characters(name):
                             def filter_character(char):
-                                if 0 < ord(char) < 128:
-                                    return char
-                                else:
-                                    self._logger.debug(
-                                        "Bad character filter %s, isalnum? %s", ord(char), char.isalnum())
-                                    return u"?"
-                            return u"".join([filter_character(char) for char in name])
-                        yield (join(*[str(filter_characters(element)) for element in file_dict[b"path"]]),
+                                if 0 < char < 128:
+                                    return chr(char)
+                                self._logger.debug("Bad character 0x%X", char)
+                                return "?"
+                            return "".join([filter_character(char) for char in name])
+                        yield (join(*[filter_characters(element) for element in file_dict[b"path"]]),
                                file_dict[b"length"])
                         continue
                     except UnicodeError:

src/tribler-core/tribler_core/modules/metadata_store/restapi/tests/test_metadata_endpoint.py

@@ -8,7 +8,6 @@
 from tribler_core.restapi.base_api_test import AbstractApiTest
 from tribler_core.tests.tools.base_test import MockObject
 from tribler_core.tests.tools.tools import timeout
-from tribler_core.tests.tools.tracker.http_tracker import HTTPTracker
 from tribler_core.tests.tools.tracker.udp_tracker import UDPTracker
 from tribler_core.utilities.random_utils import random_infohash
 from tribler_core.utilities.unicode import hexlify
@@ -203,18 +202,13 @@ async def setUp(self):
         self.udp_port = self.get_port()
         self.udp_tracker = UDPTracker(self.udp_port)
 
-        self.http_port = self.get_port()
-        self.http_tracker = HTTPTracker(self.http_port)
-
     async def tearDown(self):
         self.session.dlmgr = None
         if self.udp_tracker:
             await self.udp_tracker.stop()
-        if self.http_tracker:
-            await self.http_tracker.stop()
         await super(TestTorrentHealthEndpoint, self).tearDown()
 
-    @timeout(TORRENT_CHECK_TIMEOUT + 5)
+    @timeout(5)
     async def test_check_torrent_health(self):
         """
         Test the endpoint to fetch the health of a chant-managed, infohash-only torrent
@@ -241,20 +235,20 @@ async def test_check_torrent_health(self):
         self.session.dlmgr.dht_health_manager = MockObject()
         dht_health_dict = {"infohash": hexlify(infohash), "seeders": 1, "leechers": 2}
         self.session.dlmgr.dht_health_manager.get_health = lambda *_, **__: succeed({"DHT": [dht_health_dict]})
+        self.session.dlmgr.get_channel_downloads = lambda: []
 
         # Left for compatibility with other tests in this object
         await self.udp_tracker.start()
-        await self.http_tracker.start()
         json_response = await self.do_request(url)
         self.assertIn("health", json_response)
         self.assertIn("udp://localhost:%s" % self.udp_port, json_response['health'])
         if has_bep33_support():
             self.assertIn("DHT", json_response['health'])
 
         json_response = await self.do_request(url + '&nowait=1')
-        self.assertDictEqual(json_response, {u'checking': u'1'})
+        self.assertDictEqual(json_response, {'checking': '1'})
 
-    @timeout(TORRENT_CHECK_TIMEOUT)
+    @timeout(5)
     async def test_check_torrent_query(self):
         """
         Test that the endpoint responds with an error message if the timeout parameter has a wrong value

src/tribler-core/tribler_core/modules/tunnel/test_full_session/test_hidden_services.py

@@ -1,19 +1,15 @@
-import sys
 import time
 from asyncio import Future
-from unittest import skipIf
 
 from ipv8.messaging.anonymization.tunnel import CIRCUIT_TYPE_IP_SEEDER
 
-from tribler_common.simpledefs import DLSTATUS_SEEDING
-
 from tribler_core.modules.tunnel.test_full_session.test_tunnel_base import TestTunnelBase
 from tribler_core.tests.tools.tools import timeout
 
 
 class TestHiddenServices(TestTunnelBase):
 
-    @timeout(40)
+    @timeout(20)
     async def test_hidden_services(self):
         """
         Test the hidden services overlay by constructing an end-to-end circuit and downloading a torrent over it
@@ -25,15 +21,14 @@ async def test_hidden_services(self):
             self.assertEqual(7, len(c.get_peers()))
         self.assertEqual(7, len(self.tunnel_community_seeder.get_peers()))
 
-        test_future = Future()
+        progress = Future()
 
         def download_state_callback(ds):
             self.tunnel_community.monitor_downloads([ds])
             print(time.time(), ds.get_status(), ds.get_progress())
-            if ds.get_progress() == 1.0 and ds.get_status() == DLSTATUS_SEEDING:
-                test_future.set_result(None)
-                return 0.0
-            return 2.0
+            if ds.get_progress():
+                progress.set_result(None)
+            return 2
 
         self.tunnel_community.build_tunnels(1)
 
@@ -42,5 +37,4 @@ def download_state_callback(ds):
 
         download = self.start_anon_download(hops=1)
         download.set_state_callback(download_state_callback)
-
-        await test_future
+        await progress

src/tribler-core/tribler_core/modules/tunnel/test_full_session/test_tunnel_community.py

@@ -1,4 +1,6 @@
-from asyncio import sleep, wait_for
+from asyncio import sleep
+
+from tribler_common.simpledefs import DLSTATUS_DOWNLOADING
 
 from tribler_core.modules.tunnel.test_full_session.test_tunnel_base import TestTunnelBase
 from tribler_core.tests.tools.tools import timeout
@@ -16,28 +18,10 @@ async def test_anon_download(self):
         """
         await self.setup_nodes()
         download = self.start_anon_download()
-        self.tunnel_community.remove_circuit = lambda *_, **__: None  # Keep the circuit so we can inspect it later
-        await wait_for(download.future_finished, timeout=15)
-        self.assertGreater(self.tunnel_community.find_circuits()[0].bytes_down, 2000000)
-
-    @timeout(20)
-    async def test_anon_download_no_exits(self):
-        """
-        Testing whether an anon download does not make progress without exit nodes
-        """
-        await self.setup_nodes(num_exitnodes=0)
-        download = self.start_anon_download()
-        await sleep(10)
-        self.assertEqual(self.tunnel_community.find_circuits(), [])
-        self.assertEqual(download.get_state().get_total_transferred('down'), 0)
-
-    @timeout(20)
-    async def test_anon_download_no_relays(self):
-        """
-        Testing whether an anon download does not make progress without relay nodes
-        """
-        await self.setup_nodes(num_relays=0, num_exitnodes=1)
-        download = self.start_anon_download(hops=2)
-        await sleep(10)
-        self.assertEqual(self.tunnel_community.find_circuits(), [])
-        self.assertEqual(download.get_state().get_total_transferred('down'), 0)
+        await download.wait_for_status(DLSTATUS_DOWNLOADING)
+        self.session.dlmgr.set_download_states_callback(self.session.dlmgr.sesscb_states_callback, interval=.1)
+        while not self.tunnel_community.find_circuits():
+            await sleep(.1)
+        await sleep(.6)
+        self.assertGreater(self.tunnel_community.find_circuits()[0].bytes_up, 0)
+        self.assertGreater(self.tunnel_community.find_circuits()[0].bytes_down, 0)

src/tribler-core/tribler_core/restapi/rest_manager.py

@@ -42,7 +42,7 @@ async def error_middleware(request, handler):
     return response
 
 
-class RESTManager(TaskManager):
+class RESTManager():
     """
     This class is responsible for managing the startup and closing of the Tribler HTTP API.
     """

src/tribler-core/tribler_core/tests/tools/test_as_server.py

@@ -175,7 +175,8 @@ async def checkLoop(self, phase, *_):
         if tasks:
             self._logger.error("The event loop was dirty during %s:", phase)
         for task in tasks:
-            self._logger.error(">     %s", task)
+            name = task.get_name() if hasattr(task, 'get_name') else getattr(task, 'name', 'unknown')
+            self._logger.error(">    name=%s, %s", name, task)
 
     async def tearDown(self):
         random.seed()

src/tribler-core/tribler_core/tests/tools/tracker/udp_tracker.py

@@ -20,6 +20,10 @@ def __init__(self, tracker_session):
         self.transaction_id = -1
         self.connection_id = -1
         self.tracker_session = tracker_session
+        self.transport = None
+
+    def connection_made(self, transport):
+        self.transport = transport
 
     def datagram_received(self, response, host_and_port):
         """
@@ -56,7 +60,7 @@ def send_connection_reply(self, host, port):
         """
         self.connection_id = random.randint(0, MAX_INT32)
         response_msg = struct.pack('!iiq', TRACKER_ACTION_CONNECT, self.transaction_id, self.connection_id)
-        self.transport.write(response_msg, (host, port))
+        self.transport.sendto(response_msg, (host, port))
 
     def send_scrape_reply(self, host, port, infohashes):
         """
@@ -66,15 +70,15 @@ def send_scrape_reply(self, host, port, infohashes):
         for infohash in infohashes:
             ih_info = self.tracker_session.tracker_info.get_info_about_infohash(infohash)
             response_msg += struct.pack('!iii', ih_info['seeders'], ih_info['downloaded'], ih_info['leechers'])
-        self.transport.write(response_msg, (host, port))
+        self.transport.sendto(response_msg, (host, port))
 
     def send_error(self, host, port, error_msg):
         """
         Send an error message if the client does not follow the protocol.
         """
         response_msg = struct.pack('!ii' + str(len(error_msg)) + 's', TRACKER_ACTION_ERROR,
                                    self.transaction_id, error_msg)
-        self.transport.write(response_msg, (host, port))
+        self.transport.sendto(response_msg, (host, port))
 
 
 class UDPTracker(object):

src/tribler-gui/tribler_gui/debug_window.py

@@ -127,7 +127,7 @@ def hideEvent(self, hide_event):
     def showEvent(self, show_event):
         if self.ipv8_health_widget and self.ipv8_health_widget.isVisible():
             self.ipv8_health_widget.resume()
-            TriblerNetworkRequest("ipv8/health/enable", self.on_ipv8_health_enabled, data={"enable": True},
+            TriblerNetworkRequest("ipv8/asyncio/drift", self.on_ipv8_health_enabled, data={"enable": True},
                                   method='PUT')
 
     def run_with_timer(self, call_fn, timeout=DEBUG_PANE_REFRESH_TIMEOUT):
@@ -399,7 +399,7 @@ def load_ipv8_health_monitor(self):
             # We already loaded the widget, just resume it.
             self.ipv8_health_widget.resume()
         # Whether the widget is newly loaded or not, start the measurements.
-        TriblerNetworkRequest("ipv8/health/enable", self.on_ipv8_health_enabled, data={"enable": True}, method='PUT')
+        TriblerNetworkRequest("ipv8/asyncio/drift", self.on_ipv8_health_enabled, data={"enable": True}, method='PUT')
 
     def hide_ipv8_health_widget(self):
         """
@@ -410,7 +410,7 @@ def hide_ipv8_health_widget(self):
         """
         if self.ipv8_health_widget is not None and not self.ipv8_health_widget.is_paused:
             self.ipv8_health_widget.pause()
-            TriblerNetworkRequest("ipv8/health/enable", lambda _: None, data={"enable": False}, method='PUT')
+            TriblerNetworkRequest("ipv8/asyncio/drift", lambda _: None, data={"enable": False}, method='PUT')
 
     def on_ipv8_health(self, data):
         """
@@ -428,7 +428,7 @@ def on_ipv8_health_enabled(self, data):
         """
         if not data:
             return
-        self.run_with_timer(lambda: TriblerNetworkRequest("ipv8/health/drift", self.on_ipv8_health), 100)
+        self.run_with_timer(lambda: TriblerNetworkRequest("ipv8/asyncio/drift", self.on_ipv8_health), 100)
 
     def add_items_to_tree(self, tree, items, keys):
         tree.clear()