converter: modify for hepdata-converter v0.1.35 and new schema version

* Upgrade to latest hepdata-converter-ws-client v0.1.4.
* Pass validator_schema_version='0.1.0' converter option (closes #188).
* Also validate using old schema if converting from oldhepdata format.
* Remove this temporary fix after HEPData/hepdata-converter#32 closed.

Signed-off-by: Graeme Watt <Graeme.Watt@durham.ac.uk>

hepdata/modules/converter/views.py

@@ -57,6 +57,7 @@ def convert_endpoint():
     """
     Endpoint for general conversion, the file is passed as a GET parameter
     and options ('from=' & 'to=') are query string arguments.
+    TO DO: is this function used anywhere?  If not, it can probably be removed.
 
     :return: display_error or send_file depending on success of conversion
     """
@@ -259,6 +260,7 @@ def download_submission(submission, file_format, offline=False, force=False, riv
         'input_format': 'yaml',
         'output_format': file_format,
         'filename': 'HEPData-{0}-v{1}-{2}'.format(file_identifier, submission.version, file_format),
+        'validator_schema_version': '0.1.0',
     }
 
     if submission.doi and submission.overall_status != 'sandbox':
@@ -491,6 +493,7 @@ def download_datatable(datasubmission, file_format, *args, **kwargs):
         'output_format': file_format,
         'table': table_name,
         'filename': table_name.split('.')[0],
+        'validator_schema_version': '0.1.0',
     }
 
     hepsubmission = HEPSubmission.query.filter_by(publication_recid=datasubmission.publication_recid,

hepdata/modules/records/api.py

@@ -400,6 +400,7 @@ def process_zip_archive(file, id):
 
     if submission_found:
         basepath, submission_file_path = submission_found
+        from_oldhepdata = False
     else:
         result = check_and_convert_from_oldhepdata(file_path, id, time_stamp)
 
@@ -408,8 +409,9 @@ def process_zip_archive(file, id):
             return result
         else:
             basepath, submission_file_path = result
+            from_oldhepdata = True
 
-    return process_submission_directory(basepath, submission_file_path, id)
+    return process_submission_directory(basepath, submission_file_path, id, from_oldhepdata=from_oldhepdata)
 
 
 def check_and_convert_from_oldhepdata(input_directory, id, timestamp):

hepdata/modules/records/utils/submission.py

@@ -408,7 +408,7 @@ def _eos_fix_read_data(data_file_path):
     return data, ex
 
 
-def process_submission_directory(basepath, submission_file_path, recid, update=False, *args, **kwargs):
+def process_submission_directory(basepath, submission_file_path, recid, update=False, from_oldhepdata=False):
     """
     Goes through an entire submission directory and processes the
     files within to create DataSubmissions
@@ -418,14 +418,23 @@ def process_submission_directory(basepath, submission_file_path, recid, update=F
     :param submission_file_path:
     :param recid:
     :param update:
+    :param from_oldhepdata:
     :return:
     """
     added_file_names = []
     errors = {}
 
     if submission_file_path is not None:
 
-        submission_file_validator = SubmissionFileValidator()
+        if from_oldhepdata:
+            # The schema_version='0.1.0' argument is a temporary fix for hepdata-converter v0.1.35.
+            # Needed for "data_license: {description: null, name: null, url: null}" lines in submission.yaml.
+            # TO DO: remove the default "data_license" written to submission.yaml by the hepdata-converter.
+            # Then if using a hepdata-converter-ws Docker container, can remove this temporary fix.
+            submission_file_validator = SubmissionFileValidator(schema_version='0.1.0')
+        else:
+            submission_file_validator = SubmissionFileValidator()
+
         is_valid_submission_file = submission_file_validator.validate(file_path=submission_file_path)
 
         if is_valid_submission_file:

hepdata/version.py

@@ -30,4 +30,4 @@
 
 from __future__ import absolute_import, print_function
 
-__version__ = "0.9.4dev20200309"
+__version__ = "0.9.4dev20200319"

requirements.txt

@@ -65,11 +65,11 @@ Flask-Testing==0.4.2
 Flask-WTF==0.13.1
 funcsigs==1.0.2
 functools32==3.2.3.post2
-future==0.15.2
+future==0.18.2
 gevent==1.2.2
 greenlet==0.4.12
 gunicorn==19.5.0
-hepdata-converter-ws-client==0.1.3
+hepdata-converter-ws-client==0.1.4
 hepdata-validator==0.2.1
 idna
 IDUtils==0.2.3
@@ -145,7 +145,7 @@ PyYAML==4.2b4
 raven==6.7.0
 redis==3.2.1
 reportlab==3.1.44
-requests==2.21.0
+requests==2.23.0
 requests-oauthlib==0.7.0
 responses
 ruamel.base==1.0.0