Merge pull request #2464 from brussee/start_download_infohash

RestAPI start download endpoint

Tribler/Core/Libtorrent/LibtorrentDownloadImpl.py

@@ -178,8 +178,6 @@ def setup(self, dcfg=None, pstate=None, initialdlstatus=None,
  # Called by any thread, assume sessionlock is held
  self.set_checkpoint_disabled(checkpoint_disabled)
 
- self.set_share_mode(share_mode)
-
  try:
  # The deferred to be returned
  deferred = Deferred()
@@ -204,7 +202,7 @@ def setup(self, dcfg=None, pstate=None, initialdlstatus=None,
  def schedule_create_engine():
  self.cew_scheduled = True
  create_engine_wrapper_deferred = self.network_create_engine_wrapper(
- self.pstate_for_restart, initialdlstatus)
+ self.pstate_for_restart, initialdlstatus, share_mode=share_mode)
  create_engine_wrapper_deferred.chainDeferred(deferred)
 
 
@@ -256,7 +254,7 @@ def do_check():
  do_check()
  return can_create_deferred
 
- def network_create_engine_wrapper(self, pstate, initialdlstatus=None, checkpoint_disabled=False):
+ def network_create_engine_wrapper(self, pstate, initialdlstatus=None, checkpoint_disabled=False, share_mode=False):
  with self.dllock:
  self._logger.debug("LibtorrentDownloadImpl: network_create_engine_wrapper()")
 
@@ -268,7 +266,7 @@ def network_create_engine_wrapper(self, pstate, initialdlstatus=None, checkpoint
  atp["duplicate_is_error"] = True
  atp["hops"] = self.get_hops()
 
- if self.get_share_mode():
+ if share_mode:
  atp["flags"] = lt.add_torrent_params_flags_t.flag_share_mode
 
  self.set_checkpoint_disabled(checkpoint_disabled)
@@ -303,8 +301,8 @@ def network_create_engine_wrapper(self, pstate, initialdlstatus=None, checkpoint
  atp["name"] = self.tdef.get_name_as_unicode()
 
  self.handle = self.ltmgr.add_torrent(self, atp)
-
- if self.handle:
+ # assert self.handle.status().share_mode == share_mode
+ if self.handle.is_valid():
 
  self.set_selected_files()
 
@@ -321,11 +319,16 @@ def network_create_engine_wrapper(self, pstate, initialdlstatus=None, checkpoint
  self.handle.resolve_countries(True)
 
  else:
- self._logger.info("Could not add torrent to LibtorrentManager %s", self.tdef.get_name_as_unicode())
+ self._logger.error("Could not add torrent to LibtorrentManager %s", self.tdef.get_name_as_unicode())
+
+ self.cew_scheduled = False
+
+ # Return a deferred with the errback already being called
+ return defer.fail((self, pstate))
 
  self.cew_scheduled = False
 
- # Return a deferred with the callback already being called.
+ # Return a deferred with the callback already being called
  return defer.succeed((self, pstate))
 
  def get_anon_mode(self):
@@ -1051,7 +1054,8 @@ def restart(self, initialdlstatus=None):
 
  def schedule_create_engine(_):
  self.cew_scheduled = True
- create_engine_wrapper_deferred = self.network_create_engine_wrapper(self.pstate_for_restart, initialdlstatus)
+ create_engine_wrapper_deferred = self.network_create_engine_wrapper(
+ self.pstate_for_restart, initialdlstatus, share_mode=self.get_share_mode())
  create_engine_wrapper_deferred.addCallback(self.session.lm.on_download_wrapper_created)
 
  can_create_engine_deferred = self.can_create_engine_wrapper()

Tribler/Core/Libtorrent/LibtorrentMgr.py

@@ -10,7 +10,7 @@
 from copy import deepcopy
 from shutil import rmtree
 
-from twisted.internet import reactor
+from twisted.internet import reactor, threads
 import libtorrent as lt
 from Tribler.Core.Utilities.torrent_utils import get_info_from_handle
 from Tribler.Core.TorrentDef import TorrentDef, TorrentDefNoMetainfo
@@ -265,19 +265,19 @@ def add_torrent(self, torrentdl, atp):
 
  if infohash in self.metainfo_requests:
  self._logger.info("killing get_metainfo request for %s", infohash)
- handle = self.metainfo_requests.pop(infohash)['handle']
- if handle:
- ltsession.remove_torrent(handle, 0)
+ request_handle = self.metainfo_requests.pop(infohash)['handle']
+ if request_handle:
+ ltsession.remove_torrent(request_handle, 0)
 
- handle = ltsession.add_torrent(encode_atp(atp))
- infohash = str(handle.info_hash())
+ torrent_handle = ltsession.add_torrent(encode_atp(atp))
+ infohash = str(torrent_handle.info_hash())
  if infohash in self.torrents:
  raise DuplicateDownloadException()
  self.torrents[infohash] = (torrentdl, ltsession)
 
  self._logger.debug("added torrent %s", infohash)
 
- return handle
+ return torrent_handle
 
  def remove_torrent(self, torrentdl, removecontent=False):
  handle = torrentdl.handle
@@ -473,31 +473,24 @@ def _task_check_reachability(self):
 
  @call_on_reactor_thread
  def _schedule_next_check(self, delay, retries_left):
- self.register_task(u'check_dht', reactor.callLater(delay, self._task_check_dht, retries_left))
+ self.register_task(u'check_dht', reactor.callLater(delay, self.do_dht_check, retries_left))
 
- def _task_check_dht(self, retries_left):
+ def do_dht_check(self, retries_left):
  # Sometimes the dht fails to start. To workaround this issue we monitor the #dht_nodes, and restart if needed.
 
- def do_dht_check():
- lt_session = self.get_session()
- if lt_session:
- dht_nodes = lt_session.status().dht_nodes
- if dht_nodes <= 25:
- if dht_nodes >= 5 and retries_left > 0:
- self._logger.info(u"No enough DHT nodes %s, will try again", lt_session.status().dht_nodes)
- self._schedule_next_check(5, retries_left - 1)
- else:
- self._logger.info(u"No enough DHT nodes %s, will restart DHT", lt_session.status().dht_nodes)
- lt_session.start_dht()
- self._schedule_next_check(10, 1)
- else:
- self._logger.info("dht is working enough nodes are found (%d)", self.get_session().status().dht_nodes)
- self.dht_ready = True
- return
+ lt_session = self.get_session()
+ dht_nodes = lt_session.status().dht_nodes
+ if dht_nodes <= 25:
+ if dht_nodes >= 5 and retries_left > 0:
+ self._logger.info(u"No enough DHT nodes %s, will try again", lt_session.status().dht_nodes)
+ self._schedule_next_check(5, retries_left - 1)
  else:
+ self._logger.info(u"No enough DHT nodes %s, will restart DHT", lt_session.status().dht_nodes)
+ threads.deferToThread(lt_session.start_dht)
  self._schedule_next_check(10, 1)
-
- self.trsession.lm.threadpool.call(0, do_dht_check)
+ else:
+ self._logger.info("dht is working enough nodes are found (%d)", self.get_session().status().dht_nodes)
+ self.dht_ready = True
 
  def _map_call_on_ltsessions(self, hops, funcname, *args, **kwargs):
  if hops is None:

Tribler/Core/Modules/restapi/downloads_endpoint.py

@@ -1,9 +1,12 @@
 import json
+import os
 
 from twisted.web import http, resource
+from Tribler.Core.DownloadConfig import DownloadStartupConfig
 from Tribler.Core.Libtorrent.LibtorrentDownloadImpl import LibtorrentStatisticsResponse
+from Tribler.Core.TorrentDef import TorrentDef, TorrentDefNoMetainfo
 
-from Tribler.Core.simpledefs import DOWNLOAD, UPLOAD, dlstatus_strings
+from Tribler.Core.simpledefs import DOWNLOAD, UPLOAD, dlstatus_strings, NTFY_TORRENTS
 
 
 class DownloadBaseEndpoint(resource.Resource):
@@ -134,6 +137,42 @@ def __init__(self, session, infohash):
  self.infohash = bytes(infohash.decode('hex'))
  self.putChild("torrent", DownloadExportTorrentEndpoint(session, self.infohash))
 
+ @staticmethod
+ def create_dconfig_from_params(parameters):
+ """
+ Create a download configuration based on some given parameters. Possible parameters are:
+ - anon_hops: the number of hops for the anonymous download. 0 hops is equivalent to a plain download
+ - safe_seeding: whether the seeding of the download should be anonymous or not (0 = off, 1 = on)
+ - destination: the destination path of the torrent (where it is saved on disk)
+ """
+ download_config = DownloadStartupConfig()
+
+ anon_hops = 0
+ if 'anon_hops' in parameters and len(parameters['anon_hops']) > 0:
+ if parameters['anon_hops'][0].isdigit():
+ anon_hops = int(parameters['anon_hops'][0])
+
+ safe_seeding = False
+ if 'safe_seeding' in parameters and len(parameters['safe_seeding']) > 0 \
+ and parameters['safe_seeding'][0] == "1":
+ safe_seeding = True
+
+ if anon_hops <= 0 and safe_seeding:
+ return None, "Cannot set safe_seeding without anonymous download enabled"
+
+ if anon_hops > 0:
+ download_config.set_hops(anon_hops)
+
+ if safe_seeding:
+ download_config.set_safe_seeding(True)
+
+ if 'destination' in parameters and len(parameters['destination']) > 0:
+ if not os.path.isdir(parameters['destination'][0]):
+ return None, "Invalid destination directory specified"
+ download_config.set_dest_dir(parameters['destination'][0])
+
+ return download_config, None
+
  def render_DELETE(self, request):
  """
  .. http:delete:: /download/(string: infohash)
@@ -169,6 +208,53 @@ def render_DELETE(self, request):
 
  return json.dumps({"removed": True})
 
+ def render_PUT(self, request):
+ """
+ .. http:put:: /download/(string: infohash)
+
+ A PUT request to this endpoint will start a download from a given infohash. Metadata and peers will be fetched
+ from the libtorrent DHT. Various options can be passed:
+ - anon_hops: the number of hops for the anonymous download. 0 hops is equivalent to a plain download
+ - safe_seeding: whether the seeding of the download should be anonymous or not (0 = off, 1 = on)
+ - destination: the download destination path of the torrent
+
+ **Example request**:
+
+ .. sourcecode:: none
+
+ curl -X PUT http://localhost:8085/download/4344503b7e797ebf31582327a5baae35b11bda01
+ --data "anon_hops=2&safe_seeding=1&destination=/my/dest/on/disk/"
+
+ **Example response**:
+
+ .. sourcecode:: javascript
+
+ {"started": True}
+ """
+ parameters = http.parse_qs(request.content.read(), 1)
+
+ if self.session.has_download(self.infohash):
+ request.setResponseCode(http.CONFLICT)
+ return json.dumps({"error": "the download with the given infohash already exists"})
+
+ # Check whether we have the torrent file, otherwise, create a tdef without metainfo.
+ torrent_data = self.session.get_collected_torrent(self.infohash)
+ if torrent_data is not None:
+ tdef_download = TorrentDef.load_from_memory(torrent_data)
+ else:
+ torrent_db = self.session.open_dbhandler(NTFY_TORRENTS)
+ torrent = torrent_db.getTorrent(self.infohash, keys=['C.torrent_id', 'name'])
+ tdef_download = TorrentDefNoMetainfo(self.infohash, torrent['name'])
+
+ download_config, error = DownloadSpecificEndpoint.create_dconfig_from_params(parameters)
+ if not error:
+ self.session.start_download_from_tdef(tdef_download, download_config)
+ else:
+ request.setResponseCode(http.BAD_REQUEST)
+ return json.dumps({"error": error})
+
+ return json.dumps({"started": True})
+
  def render_PATCH(self, request):
  """
  .. http:patch:: /download/(string: infohash)

Tribler/Test/Core/Libtorrent/test_libtorrent_download_impl.py

@@ -97,29 +97,44 @@ def test_restart(self):
  tdef = self.create_tdef()
 
  impl = LibtorrentDownloadImpl(self.session, tdef)
- impl.handle = None
+ impl.handle = MockObject()
+ impl.handle.set_priority = lambda _: None
+ impl.handle.set_sequential_download = lambda _: None
+ impl.handle.resume = lambda: None
+ impl.handle.status = lambda: fake_status
+ fake_status = MockObject()
+ fake_status.share_mode = False
  # Create a dummy download config
  impl.dlconfig = DownloadStartupConfig().dlconfig.copy()
  impl.session.lm.on_download_wrapper_created = lambda _: True
  impl.restart()
 
  @deferred(timeout=20)
  def test_multifile_torrent(self):
- t = TorrentDef()
+ tdef = TorrentDef()
 
  dn = os.path.join(TESTS_DATA_DIR, "contentdir")
- t.add_content(dn, "dirintorrent")
+ tdef.add_content(dn, "dirintorrent")
 
  fn = os.path.join(TESTS_DATA_DIR, "video.avi")
- t.add_content(fn, os.path.join("dirintorrent", "video.avi"))
+ tdef.add_content(fn, os.path.join("dirintorrent", "video.avi"))
 
- t.set_tracker("http://tribler.org/announce")
- t.finalize()
+ tdef.set_tracker("http://tribler.org/announce")
+ tdef.finalize()
 
- impl = LibtorrentDownloadImpl(self.session, t)
- # Override the addtorrent because it will be called
- impl.ltmgr = self.session.lm.ltmgr
- impl.ltmgr.add_torrent = lambda ignored, ignored2: False
+ impl = LibtorrentDownloadImpl(self.session, tdef)
+ # Override the add_torrent because it will be called
+ impl.ltmgr = MockObject()
+ impl.ltmgr.add_torrent = lambda _, _dummy2: fake_handler
+ impl.set_selected_files = lambda: None
+ fake_handler = MockObject()
+ fake_handler.is_valid = lambda: True
+ fake_handler.status = lambda: fake_status
+ fake_handler.set_share_mode = lambda _: None
+ fake_handler.resume = lambda: None
+ fake_handler.resolve_countries = lambda _: None
+ fake_status = MockObject()
+ fake_status.share_mode = False
  # Create a dummy download config
  impl.dlconfig = DownloadStartupConfig().dlconfig.copy()
  # Create a dummy pstate
@@ -362,7 +377,6 @@ def test_setup_exception(self):
  """
  Testing whether an exception in the setup method of LibtorrentDownloadImpl is handled correctly
  """
- self.libtorrent_download_impl.handle.set_share_mode = lambda _: None
  self.libtorrent_download_impl.setup()
  self.assertIsInstance(self.libtorrent_download_impl.error, Exception)