Merge pull request #153 from Prodiguer/robust2ways

Two bug fixes to prevent crashes

sdt/bin/sdenqueue.py

@@ -139,7 +139,13 @@ def add_file(f):
  f.status=sdconst.TRANSFER_STATUS_WAITING
  f.crea_date=sdtime.now()
 
- sdfiledao.add_file(f,commit=False)
+ try:
+ sdfiledao.add_file(f,commit=False)
+ except Exception as e:
+ sdlog.error("SDENQUEU-005","Failed to create transfer (local_path=%s,url=%s)"%(f.get_full_local_path(),f.url))
+ sdlog.error("SDENQUEU-006","Exception was %s"%e)
+ sdlog.error("SDENQUEU-007","This file was found from the search url %s"
+ % getattr(f, 'search_url', '(unknown)') )
 
 def add_dataset(f):
  """

sdt/bin/sdjson.py

@@ -25,7 +25,7 @@ def parse_parameters(buffer):
  raise
 
  params={}
- footer_node=xmldoc["facet_counts"]
+ footer_node=xmldoc.get("facet_counts",{"facet_fields":{}})
  fields_node=footer_node["facet_fields"]
  for facet_name,li in fields_node.iteritems():
  items=[]
@@ -74,10 +74,10 @@ def parse_metadata(buffer):
 
  # retrieve header & footer (those nodes always exist)
  header_node=xmldoc["responseHeader"]
- footer_node=xmldoc["facet_counts"]
+ # not used: footer_node=xmldoc["facet_counts"]
 
  # parse footer
- fields_node=footer_node["facet_fields"]
+ # not used: fields_node=footer_node["facet_fields"]
 
  # --- parse body node --- #
 

sdt/bin/sdnetutils.py

@@ -70,6 +70,8 @@ def call_web_service(url,timeout=sdconst.SEARCH_API_HTTP_TIMEOUT,lowmem=False):
  raise SDException('SDNETUTI-008','Network error (see log for details)') # we raise a new exception 'network error' here, because most of the time, 'xml parsing error' is due to an 'network error'.
 
  sdlog.debug("SDNETUTI-044","files-count=%d"%len(di.get('files')))
+ for difile in di['files']:
+ difile['search_url'] = url
 
  return sdtypes.Response(call_duration=elapsed_time,lowmem=lowmem,**di) # RAM storage is ok here as one response is limited by SEARCH_API_CHUNKSIZE