Merge branch 'main' into observer-role

.github/dependabot.yml

@@ -21,3 +21,11 @@ updates:
     ignore:
       - dependency-name: "pytest"  # see https://github.com/HEPData/hepdata/issues/815
       - dependency-name: "pytest-cov"  # see https://github.com/HEPData/hepdata/issues/580
+      - dependency-name: "invenio-access"  # see https://github.com/HEPData/hepdata/issues/848
+      - dependency-name: "invenio-assets"  # see https://github.com/HEPData/hepdata/issues/848
+      - dependency-name: "invenio-logging"  # see https://github.com/HEPData/hepdata/issues/848
+      - dependency-name: "invenio-oauthclient"  # see https://github.com/HEPData/hepdata/issues/848
+      - dependency-name: "invenio-pidstore"  # see https://github.com/HEPData/hepdata/issues/848
+      - dependency-name: "invenio-records"  # see https://github.com/HEPData/hepdata/issues/848
+      - dependency-name: "invenio-search"  # see https://github.com/HEPData/hepdata/issues/848
+      - dependency-name: "invenio-userprofiles"  # see https://github.com/HEPData/hepdata/issues/848

INSTALL.rst

@@ -249,7 +249,7 @@ Now start the HEPData web application in debug mode:
 
 Then open your preferred web browser (Chrome, Firefox, Safari, etc.) at http://localhost:5000/ .
 
-On macOS Monterey you might find that ControlCenter is already listening to port 5000
+On macOS Monterey (and later) you might find that ControlCenter is already listening to port 5000
 (check with ``lsof -i -P | grep 5000``).  If this is the case,
 `turn off AirPlay Receiver <https://support.apple.com/en-gb/guide/mac-help/mchl15c9e4b5/12.0/mac/12.0>`_.
 

hepdata/cli.py

@@ -223,9 +223,9 @@ def do_unload(records_to_unload):
 
 @utils.command()
 @with_appcontext
-@click.option('--endpoint', '-e', type=str, help='Specific endpoint to update (e.g. "rivet" or "MadAnalysis" or "SModelS"). Omit for all.')
+@click.option('--endpoint', '-e', type=str, help='Specific endpoint to update (e.g. "rivet" or "MadAnalysis" or "SModelS" or "Combine"). Omit for all.')
 def find_and_add_record_analyses(endpoint):
-    """Finds analyses such as Rivet, MadAnalysis 5 and SModelS and adds them to records."""
+    """Finds analyses such as Rivet, MadAnalysis 5, SModelS and Combine and adds them to records."""
     update_analyses(endpoint)
 
 

hepdata/config.py

@@ -332,7 +332,16 @@ def _(x):
         'endpoint_url': 'https://zenodo.org/records/13952092/files/smodels-analyses.hepdata.json?download=1',
         'url_template': '{0}',
         'subscribe_user_id': 7766
-    }
+    },
+    'Combine': {
+        'endpoint_url': 'https://cms-public-likelihoods-list.web.cern.ch/artifacts/output.json',
+        'url_template': 'https://doi.org/{0}',
+        'description': 'Statistical models',
+        'license': {
+            'name': 'cc-by-4.0',
+            'url': 'https://creativecommons.org/licenses/by/4.0'
+         },
+    },
     #'ufo': {},
     #'xfitter': {},
     #'applgrid': {},

hepdata/ext/opensearch/document_enhancers.py

@@ -94,7 +94,7 @@ def add_shortened_authors(doc):
 
 def add_analyses(doc):
     """
-    Add analyses links such as Rivet, MadAnalysis 5, SModelS, HistFactory and NUISANCE to the index.
+    Add analyses links such as Rivet, MadAnalysis 5, SModelS, Combine, HistFactory and NUISANCE to the index.
 
     :param doc:
     :return:

hepdata/modules/records/assets/js/hepdata_common.js

@@ -46,6 +46,7 @@ HEPDATA.file_type_to_details = {
   "rivet": {"icon": "area-chart", "description": "Rivet Analysis"},
   "madanalysis": {"icon": "area-chart", "description": "MadAnalysis 5 Analysis"},
   "smodels": {"icon": "area-chart", "description": "SModelS Analysis"},
+  "combine": {"icon": "area-chart", "description": "Combine Analysis"},
   "xfitter": {"icon": "area-chart", "description": "xFitter Analysis"},
   "applgrid": {"icon": "area-chart", "description": "APPLgrid Analysis"},
   "ufo": {"icon": "rocket", "description": "Universal Feynrules Output (UFO)"},

hepdata/modules/records/templates/hepdata_records/components/resources-widget.html

@@ -41,6 +41,7 @@ <h4>Add Resource for <span id="selected_resource_item">Submission</span></h4>
                                                     <option value="applgrid">APPLgrid</option>
                                                     <option value="MadAnalysis">MadAnalysis 5</option>
                                                     <option value="SModelS">SModelS</option>
+                                                    <option value="Combine">Combine</option>
                                                     <option value="rivet">Rivet</option>
                                                     <option value="fastnlo">fastNLO</option>
                                                     <option value="ufo">Universal Feynrules Output (UFO)</option>

hepdata/modules/records/utils/analyses.py

@@ -35,6 +35,7 @@
 from hepdata.utils.users import get_user_from_id
 from hepdata.modules.records.subscribers.rest import subscribe
 from hepdata.modules.records.subscribers.api import is_current_user_subscribed_to_record
+from hepdata.modules.records.utils.common import get_license
 
 logging.basicConfig()
 log = logging.getLogger(__name__)
@@ -43,10 +44,11 @@
 @shared_task
 def update_analyses(endpoint=None):
     """
-    Update (Rivet, MadAnalysis 5 and SModelS) analyses and remove outdated resources.
+    Update (Rivet, MadAnalysis 5, SModelS and Combine) analyses and remove outdated resources.
     Allow bulk subscription to record update notifications if "subscribe_user_id" in endpoint.
+    Add optional "description" and "license" fields if present in endpoint.
 
-    :param endpoint: either "rivet" or "MadAnalysis" or "SModelS" or None (default) for both
+    :param endpoint: either "rivet" or "MadAnalysis" or "SModelS" or "Combine" or None (default) for both
     """
     endpoints = current_app.config["ANALYSES_ENDPOINTS"]
     for analysis_endpoint in endpoints:
@@ -86,6 +88,13 @@ def update_analyses(endpoint=None):
                                     file_location=_resource_url,
                                     file_type=analysis_endpoint)
 
+                                if "description" in endpoints[analysis_endpoint]:
+                                    new_resource.file_description = str(endpoints[analysis_endpoint]["description"])
+
+                                if "license" in endpoints[analysis_endpoint]:
+                                    resource_license = get_license(endpoints[analysis_endpoint]["license"])
+                                    new_resource.file_license = resource_license.id
+
                                 submission.resources.append(new_resource)
                                 num_new_resources += 1
 

hepdata/modules/search/templates/hepdata_search/modals/search_help.html

@@ -243,6 +243,13 @@ <h4>Other useful searches</h4>
                                        (SModelS analysis)
                                     </span>
                                 </li>
+                                <li>
+                                    <a href='/search?q=analysis:Combine&sort_by=latest'
+                                       target="_new">analysis:Combine</a>
+                                    <span class="text-muted">
+                                       (CMS statistical models in Combine format)
+                                    </span>
+                                </li>
                                 <li>
                                     <a href='/search?q=analysis:HistFactory&sort_by=latest'
                                        target="_new">analysis:HistFactory</a>

hepdata/version.py

@@ -28,4 +28,4 @@
 and parsed by ``setup.py``.
 """
 
-__version__ = "0.9.4dev20241112"
+__version__ = "0.9.4dev20241211"

requirements.txt

@@ -1,22 +1,18 @@
 beautifulsoup4==4.12.3
 bleach==6.2.0
-datacite==1.1.4
+datacite==1.2.0
 gunicorn==23.0.0
 hepdata-converter-ws-client==0.2.2
 hepdata-validator==0.3.5
-invenio-access==2.0.0     # Indirect (needed by invenio-admin)
-invenio-accounts==5.1.3
-invenio-admin==1.5.1
-invenio-assets==3.0.3
+invenio-access[admin]==3.0.2
+invenio-assets==3.1.0
 invenio-config==1.0.4
-invenio-db[postgresql]==1.1.5
 invenio-logging[sentry_sdk]==2.1.1
-invenio-oauthclient==4.0.2
-invenio-pidstore==1.3.1
-invenio-records==2.3.0
+invenio-oauthclient==4.1.3
+invenio-pidstore==1.3.4
+invenio-records[postgresql]==2.4.1
 invenio-search[opensearch2]==2.4.1
-invenio-theme==3.4.3
-invenio-userprofiles==3.0.0
+invenio-userprofiles==3.0.1
 python-twitter-v2==0.9.2
 responses==0.25.3
 unicodeit==0.7.5

tests/records_test.py

@@ -1031,7 +1031,7 @@ def test_create_breadcrumb_text():
 
 
 def test_update_analyses(app):
-    """ Test update of Rivet, MadAnalyses 5 and SModelS analyses """
+    """ Test update of Rivet, MadAnalyses 5, SModelS and Combine analyses """
 
     # Import a record that already has a Rivet analysis attached (but with '#' in the URL)
     import_records(['ins1203852'], synchronous=True)
@@ -1074,6 +1074,23 @@ def test_update_analyses(app):
     submission = get_latest_hepsubmission(inspire_id='1847779', overall_status='finished')
     assert is_current_user_subscribed_to_record(submission.publication_recid, user)
 
+    # Import a record that has an associated Combine analysis
+    import_records(['ins2796231'], synchronous=True)
+    analysis_resources = DataResource.query.filter_by(file_type='Combine').all()
+    assert len(analysis_resources) == 0
+    analysis_resources = DataResource.query.filter_by(file_location='https://doi.org/10.17181/bp9fx-6qs64').all()
+    assert len(analysis_resources) == 1
+    db.session.delete(analysis_resources[0])  # delete resource so it can be re-added in next step
+    db.session.commit()
+    update_analyses('Combine')
+    analysis_resources = DataResource.query.filter_by(file_type='Combine').all()
+    assert len(analysis_resources) == 1
+    assert analysis_resources[0].file_location == 'https://doi.org/10.17181/bp9fx-6qs64'
+    assert analysis_resources[0].file_description == 'Statistical models'
+    license_data = License.query.filter_by(id=analysis_resources[0].file_license).first()
+    assert license_data.name == 'cc-by-4.0'
+    assert license_data.url == 'https://creativecommons.org/licenses/by/4.0'
+
 
 def test_generate_license_data_by_id(app):
     """

tests/search_test.py

@@ -21,14 +21,16 @@
 from opensearch_dsl import Search, Index
 from datetime import datetime
 import pytest
+import os as op_s
 from invenio_db import db
 from unittest.mock import call
 
 from hepdata.ext.opensearch.config.os_config import \
     add_default_aggregations, sort_fields_mapping
 from hepdata.ext.opensearch import api as os_api
 from hepdata.ext.opensearch.config.os_config import get_filter_field
-from hepdata.ext.opensearch.document_enhancers import add_data_keywords, process_cmenergies
+from hepdata.ext.opensearch.document_enhancers import add_data_keywords, process_cmenergies, add_analyses
+from hepdata.modules.records.utils.submission import process_submission_directory
 from hepdata.utils.miscellaneous import get_resource_data
 from hepdata.ext.opensearch.process_results import merge_results, match_tables_to_papers, \
     get_basic_record_information, is_datatable
@@ -753,6 +755,88 @@ def test_add_data_keywords():
     assert 'NOTAREALKEYWORD' not in doc['data_keywords']
 
 
+def test_add_analyses(app):
+    """
+    Tests the add_analyses function to ensure that DataSubmission data
+        is properly added to the doc object during document enhancement.
+
+    Currently testing against: NUISANCE, HistFactory, MadAnalysis
+    """
+    # Here, test_data should match the contents of the test_folder
+    test_folder = "test_data/test_analysis_submission"
+    test_data = [
+        {  # ProSelecta/NUISANCE
+            "type": "NUISANCE",
+            "filename": "test.ProSelecta"
+        },
+        {  # HistFactory entry
+            "type": "HistFactory",
+            "filename": "test.tar.gz"
+        },
+    ]
+    # This should probably be changed to use SITE_URL or some similar concept
+    analysis_url = "http://localhost:5000/record/resource/%s?landing_page=true"
+
+    with app.app_context():
+        # Creating and submitting the test submission containing resources
+        # op_s is os module
+        base_dir = op_s.path.dirname(op_s.path.realpath(__file__))
+
+        hepsubmission = HEPSubmission(publication_recid=123456,
+                                      overall_status="finished",
+                                      version=1,
+                                      doi="10.17182/hepdata.123456")
+        db.session.add(hepsubmission)
+        db.session.commit()
+
+        # Setting directory and executing processing
+        directory = op_s.path.join(base_dir, test_folder)
+        errors = process_submission_directory(
+            directory,
+            op_s.path.join(directory, "submission.yaml"),
+            hepsubmission.publication_recid
+        )
+
+        # No errors should happen
+        assert not errors
+
+        # Add MadAnalysis DataResource object separately
+        mad_analysis_resource = DataResource(
+            file_location = "placeholder",
+            file_type = "MadAnalysis",
+            file_description = "placeholder"
+        )
+
+        # Adding object to database
+        hepsubmission.resources.append(mad_analysis_resource)
+        db.session.add(mad_analysis_resource)
+        db.session.add(hepsubmission)
+        db.session.commit()
+
+        # Set up a generic doc object to match what add_analyses expects
+        test_doc = {"analyses": [], "recid": hepsubmission.publication_recid}
+        # Run the test add_analyses function
+        add_analyses(test_doc)
+
+        # A sorted list of all DataResource object IDs from submission
+        data_ids = sorted([r.id for r in hepsubmission.resources])
+
+        # There should be 3 analyses and 3 resources
+        assert len(data_ids) == len(test_doc["analyses"]) == 3
+
+        # There should be one entry into test_data per resource ID
+        # Looping through the test, resource IDs and the analysis outputs
+        for test, d_id, analysis in zip(test_data, data_ids, test_doc["analyses"]):
+            # Set the expected ID in the url to the sorted data_id entry
+            test["analysis"] = (analysis_url % d_id)
+            # Confirm data has been added to the doc
+            assert analysis == test
+
+        # Checking MadAnalysis added after submission
+        mad_analysis = test_doc["analyses"][-1]
+        assert mad_analysis["type"] == "MadAnalysis"
+
+
 def test_process_cmenergies():
     test_keywords = {
         "cmenergies": [

tests/test_data/test_analysis_submission/data1.yaml

@@ -0,0 +1,10 @@
+dependent_variables:
+- header: {name: TestData1-dependent-V1, units: NA}
+  qualifiers:
+  - {name: TestData1-qualifier-V1, value: 0}
+  values:
+    - {value: 0}
+independent_variables:
+- header: {name: TestData1-independent-V1, units: NA}
+  values:
+  - value: 0

tests/test_data/test_analysis_submission/submission.yaml

@@ -0,0 +1,12 @@
+---
+description: "TestSubmission1-V1"
+comment: TestComment1-V1
+additional_resources:
+- {description: Test, location: test.ProSelecta,  type: ProSelecta }
+- {description: Test, location: test.tar.gz, type: HistFactory }
+---
+name: "TestTable1-V1"
+description: TestTable1-description-V1
+keywords:
+- {name: cmenergies, values: [0]}
+data_file: data1.yaml