diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d26945eb..551371cf 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -21,3 +21,11 @@ updates: ignore: - dependency-name: "pytest" # see https://github.com/HEPData/hepdata/issues/815 - dependency-name: "pytest-cov" # see https://github.com/HEPData/hepdata/issues/580 + - dependency-name: "invenio-access" # see https://github.com/HEPData/hepdata/issues/848 + - dependency-name: "invenio-assets" # see https://github.com/HEPData/hepdata/issues/848 + - dependency-name: "invenio-logging" # see https://github.com/HEPData/hepdata/issues/848 + - dependency-name: "invenio-oauthclient" # see https://github.com/HEPData/hepdata/issues/848 + - dependency-name: "invenio-pidstore" # see https://github.com/HEPData/hepdata/issues/848 + - dependency-name: "invenio-records" # see https://github.com/HEPData/hepdata/issues/848 + - dependency-name: "invenio-search" # see https://github.com/HEPData/hepdata/issues/848 + - dependency-name: "invenio-userprofiles" # see https://github.com/HEPData/hepdata/issues/848 diff --git a/INSTALL.rst b/INSTALL.rst index a6550d76..597e341e 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -249,7 +249,7 @@ Now start the HEPData web application in debug mode: Then open your preferred web browser (Chrome, Firefox, Safari, etc.) at http://localhost:5000/ . -On macOS Monterey you might find that ControlCenter is already listening to port 5000 +On macOS Monterey (and later) you might find that ControlCenter is already listening to port 5000 (check with ``lsof -i -P | grep 5000``). If this is the case, `turn off AirPlay Receiver `_. diff --git a/hepdata/cli.py b/hepdata/cli.py index 36b4d54f..43849f50 100644 --- a/hepdata/cli.py +++ b/hepdata/cli.py @@ -223,9 +223,9 @@ def do_unload(records_to_unload): @utils.command() @with_appcontext -@click.option('--endpoint', '-e', type=str, help='Specific endpoint to update (e.g. "rivet" or "MadAnalysis" or "SModelS"). Omit for all.') +@click.option('--endpoint', '-e', type=str, help='Specific endpoint to update (e.g. "rivet" or "MadAnalysis" or "SModelS" or "Combine"). Omit for all.') def find_and_add_record_analyses(endpoint): - """Finds analyses such as Rivet, MadAnalysis 5 and SModelS and adds them to records.""" + """Finds analyses such as Rivet, MadAnalysis 5, SModelS and Combine and adds them to records.""" update_analyses(endpoint) diff --git a/hepdata/config.py b/hepdata/config.py index e80ad50a..9925a619 100644 --- a/hepdata/config.py +++ b/hepdata/config.py @@ -332,7 +332,16 @@ def _(x): 'endpoint_url': 'https://zenodo.org/records/13952092/files/smodels-analyses.hepdata.json?download=1', 'url_template': '{0}', 'subscribe_user_id': 7766 - } + }, + 'Combine': { + 'endpoint_url': 'https://cms-public-likelihoods-list.web.cern.ch/artifacts/output.json', + 'url_template': 'https://doi.org/{0}', + 'description': 'Statistical models', + 'license': { + 'name': 'cc-by-4.0', + 'url': 'https://creativecommons.org/licenses/by/4.0' + }, + }, #'ufo': {}, #'xfitter': {}, #'applgrid': {}, diff --git a/hepdata/ext/opensearch/document_enhancers.py b/hepdata/ext/opensearch/document_enhancers.py index 6ca9173a..f5554b8b 100644 --- a/hepdata/ext/opensearch/document_enhancers.py +++ b/hepdata/ext/opensearch/document_enhancers.py @@ -94,7 +94,7 @@ def add_shortened_authors(doc): def add_analyses(doc): """ - Add analyses links such as Rivet, MadAnalysis 5, SModelS, HistFactory and NUISANCE to the index. + Add analyses links such as Rivet, MadAnalysis 5, SModelS, Combine, HistFactory and NUISANCE to the index. :param doc: :return: diff --git a/hepdata/modules/records/assets/js/hepdata_common.js b/hepdata/modules/records/assets/js/hepdata_common.js index a55d6963..0aec95fb 100644 --- a/hepdata/modules/records/assets/js/hepdata_common.js +++ b/hepdata/modules/records/assets/js/hepdata_common.js @@ -46,6 +46,7 @@ HEPDATA.file_type_to_details = { "rivet": {"icon": "area-chart", "description": "Rivet Analysis"}, "madanalysis": {"icon": "area-chart", "description": "MadAnalysis 5 Analysis"}, "smodels": {"icon": "area-chart", "description": "SModelS Analysis"}, + "combine": {"icon": "area-chart", "description": "Combine Analysis"}, "xfitter": {"icon": "area-chart", "description": "xFitter Analysis"}, "applgrid": {"icon": "area-chart", "description": "APPLgrid Analysis"}, "ufo": {"icon": "rocket", "description": "Universal Feynrules Output (UFO)"}, diff --git a/hepdata/modules/records/templates/hepdata_records/components/resources-widget.html b/hepdata/modules/records/templates/hepdata_records/components/resources-widget.html index 17c295b5..328a5be1 100644 --- a/hepdata/modules/records/templates/hepdata_records/components/resources-widget.html +++ b/hepdata/modules/records/templates/hepdata_records/components/resources-widget.html @@ -41,6 +41,7 @@

Add Resource for Submission

+ diff --git a/hepdata/modules/records/utils/analyses.py b/hepdata/modules/records/utils/analyses.py index 35b4b730..5219e01a 100644 --- a/hepdata/modules/records/utils/analyses.py +++ b/hepdata/modules/records/utils/analyses.py @@ -35,6 +35,7 @@ from hepdata.utils.users import get_user_from_id from hepdata.modules.records.subscribers.rest import subscribe from hepdata.modules.records.subscribers.api import is_current_user_subscribed_to_record +from hepdata.modules.records.utils.common import get_license logging.basicConfig() log = logging.getLogger(__name__) @@ -43,10 +44,11 @@ @shared_task def update_analyses(endpoint=None): """ - Update (Rivet, MadAnalysis 5 and SModelS) analyses and remove outdated resources. + Update (Rivet, MadAnalysis 5, SModelS and Combine) analyses and remove outdated resources. Allow bulk subscription to record update notifications if "subscribe_user_id" in endpoint. + Add optional "description" and "license" fields if present in endpoint. - :param endpoint: either "rivet" or "MadAnalysis" or "SModelS" or None (default) for both + :param endpoint: either "rivet" or "MadAnalysis" or "SModelS" or "Combine" or None (default) for both """ endpoints = current_app.config["ANALYSES_ENDPOINTS"] for analysis_endpoint in endpoints: @@ -86,6 +88,13 @@ def update_analyses(endpoint=None): file_location=_resource_url, file_type=analysis_endpoint) + if "description" in endpoints[analysis_endpoint]: + new_resource.file_description = str(endpoints[analysis_endpoint]["description"]) + + if "license" in endpoints[analysis_endpoint]: + resource_license = get_license(endpoints[analysis_endpoint]["license"]) + new_resource.file_license = resource_license.id + submission.resources.append(new_resource) num_new_resources += 1 diff --git a/hepdata/modules/search/templates/hepdata_search/modals/search_help.html b/hepdata/modules/search/templates/hepdata_search/modals/search_help.html index 58e3410d..9feddd01 100644 --- a/hepdata/modules/search/templates/hepdata_search/modals/search_help.html +++ b/hepdata/modules/search/templates/hepdata_search/modals/search_help.html @@ -243,6 +243,13 @@

Other useful searches

(SModelS analysis) +
  • + analysis:Combine + + (CMS statistical models in Combine format) + +
  • analysis:HistFactory diff --git a/hepdata/version.py b/hepdata/version.py index eeef8cfc..22e8d161 100644 --- a/hepdata/version.py +++ b/hepdata/version.py @@ -28,4 +28,4 @@ and parsed by ``setup.py``. """ -__version__ = "0.9.4dev20241112" +__version__ = "0.9.4dev20241211" diff --git a/requirements.txt b/requirements.txt index 0e6c00fc..11d52b2e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,22 +1,18 @@ beautifulsoup4==4.12.3 bleach==6.2.0 -datacite==1.1.4 +datacite==1.2.0 gunicorn==23.0.0 hepdata-converter-ws-client==0.2.2 hepdata-validator==0.3.5 -invenio-access==2.0.0 # Indirect (needed by invenio-admin) -invenio-accounts==5.1.3 -invenio-admin==1.5.1 -invenio-assets==3.0.3 +invenio-access[admin]==3.0.2 +invenio-assets==3.1.0 invenio-config==1.0.4 -invenio-db[postgresql]==1.1.5 invenio-logging[sentry_sdk]==2.1.1 -invenio-oauthclient==4.0.2 -invenio-pidstore==1.3.1 -invenio-records==2.3.0 +invenio-oauthclient==4.1.3 +invenio-pidstore==1.3.4 +invenio-records[postgresql]==2.4.1 invenio-search[opensearch2]==2.4.1 -invenio-theme==3.4.3 -invenio-userprofiles==3.0.0 +invenio-userprofiles==3.0.1 python-twitter-v2==0.9.2 responses==0.25.3 unicodeit==0.7.5 diff --git a/tests/records_test.py b/tests/records_test.py index 988f7692..2f6b4a14 100644 --- a/tests/records_test.py +++ b/tests/records_test.py @@ -1031,7 +1031,7 @@ def test_create_breadcrumb_text(): def test_update_analyses(app): - """ Test update of Rivet, MadAnalyses 5 and SModelS analyses """ + """ Test update of Rivet, MadAnalyses 5, SModelS and Combine analyses """ # Import a record that already has a Rivet analysis attached (but with '#' in the URL) import_records(['ins1203852'], synchronous=True) @@ -1074,6 +1074,23 @@ def test_update_analyses(app): submission = get_latest_hepsubmission(inspire_id='1847779', overall_status='finished') assert is_current_user_subscribed_to_record(submission.publication_recid, user) + # Import a record that has an associated Combine analysis + import_records(['ins2796231'], synchronous=True) + analysis_resources = DataResource.query.filter_by(file_type='Combine').all() + assert len(analysis_resources) == 0 + analysis_resources = DataResource.query.filter_by(file_location='https://doi.org/10.17181/bp9fx-6qs64').all() + assert len(analysis_resources) == 1 + db.session.delete(analysis_resources[0]) # delete resource so it can be re-added in next step + db.session.commit() + update_analyses('Combine') + analysis_resources = DataResource.query.filter_by(file_type='Combine').all() + assert len(analysis_resources) == 1 + assert analysis_resources[0].file_location == 'https://doi.org/10.17181/bp9fx-6qs64' + assert analysis_resources[0].file_description == 'Statistical models' + license_data = License.query.filter_by(id=analysis_resources[0].file_license).first() + assert license_data.name == 'cc-by-4.0' + assert license_data.url == 'https://creativecommons.org/licenses/by/4.0' + def test_generate_license_data_by_id(app): """ diff --git a/tests/search_test.py b/tests/search_test.py index 72131fa2..f54d67e0 100644 --- a/tests/search_test.py +++ b/tests/search_test.py @@ -21,6 +21,7 @@ from opensearch_dsl import Search, Index from datetime import datetime import pytest +import os as op_s from invenio_db import db from unittest.mock import call @@ -28,7 +29,8 @@ add_default_aggregations, sort_fields_mapping from hepdata.ext.opensearch import api as os_api from hepdata.ext.opensearch.config.os_config import get_filter_field -from hepdata.ext.opensearch.document_enhancers import add_data_keywords, process_cmenergies +from hepdata.ext.opensearch.document_enhancers import add_data_keywords, process_cmenergies, add_analyses +from hepdata.modules.records.utils.submission import process_submission_directory from hepdata.utils.miscellaneous import get_resource_data from hepdata.ext.opensearch.process_results import merge_results, match_tables_to_papers, \ get_basic_record_information, is_datatable @@ -753,6 +755,88 @@ def test_add_data_keywords(): assert 'NOTAREALKEYWORD' not in doc['data_keywords'] +def test_add_analyses(app): + """ + Tests the add_analyses function to ensure that DataSubmission data + is properly added to the doc object during document enhancement. + + Currently testing against: NUISANCE, HistFactory, MadAnalysis + """ + # Here, test_data should match the contents of the test_folder + test_folder = "test_data/test_analysis_submission" + test_data = [ + { # ProSelecta/NUISANCE + "type": "NUISANCE", + "filename": "test.ProSelecta" + }, + { # HistFactory entry + "type": "HistFactory", + "filename": "test.tar.gz" + }, + ] + # This should probably be changed to use SITE_URL or some similar concept + analysis_url = "http://localhost:5000/record/resource/%s?landing_page=true" + + with app.app_context(): + # Creating and submitting the test submission containing resources + # op_s is os module + base_dir = op_s.path.dirname(op_s.path.realpath(__file__)) + + hepsubmission = HEPSubmission(publication_recid=123456, + overall_status="finished", + version=1, + doi="10.17182/hepdata.123456") + db.session.add(hepsubmission) + db.session.commit() + + # Setting directory and executing processing + directory = op_s.path.join(base_dir, test_folder) + errors = process_submission_directory( + directory, + op_s.path.join(directory, "submission.yaml"), + hepsubmission.publication_recid + ) + + # No errors should happen + assert not errors + + # Add MadAnalysis DataResource object separately + mad_analysis_resource = DataResource( + file_location = "placeholder", + file_type = "MadAnalysis", + file_description = "placeholder" + ) + + # Adding object to database + hepsubmission.resources.append(mad_analysis_resource) + db.session.add(mad_analysis_resource) + db.session.add(hepsubmission) + db.session.commit() + + # Set up a generic doc object to match what add_analyses expects + test_doc = {"analyses": [], "recid": hepsubmission.publication_recid} + # Run the test add_analyses function + add_analyses(test_doc) + + # A sorted list of all DataResource object IDs from submission + data_ids = sorted([r.id for r in hepsubmission.resources]) + + # There should be 3 analyses and 3 resources + assert len(data_ids) == len(test_doc["analyses"]) == 3 + + # There should be one entry into test_data per resource ID + # Looping through the test, resource IDs and the analysis outputs + for test, d_id, analysis in zip(test_data, data_ids, test_doc["analyses"]): + # Set the expected ID in the url to the sorted data_id entry + test["analysis"] = (analysis_url % d_id) + # Confirm data has been added to the doc + assert analysis == test + + # Checking MadAnalysis added after submission + mad_analysis = test_doc["analyses"][-1] + assert mad_analysis["type"] == "MadAnalysis" + + def test_process_cmenergies(): test_keywords = { "cmenergies": [ diff --git a/tests/test_data/test_analysis_submission/data1.yaml b/tests/test_data/test_analysis_submission/data1.yaml new file mode 100644 index 00000000..ec9813a1 --- /dev/null +++ b/tests/test_data/test_analysis_submission/data1.yaml @@ -0,0 +1,10 @@ +dependent_variables: +- header: {name: TestData1-dependent-V1, units: NA} + qualifiers: + - {name: TestData1-qualifier-V1, value: 0} + values: + - {value: 0} +independent_variables: +- header: {name: TestData1-independent-V1, units: NA} + values: + - value: 0 \ No newline at end of file diff --git a/tests/test_data/test_analysis_submission/submission.yaml b/tests/test_data/test_analysis_submission/submission.yaml new file mode 100644 index 00000000..44667cb8 --- /dev/null +++ b/tests/test_data/test_analysis_submission/submission.yaml @@ -0,0 +1,12 @@ +--- +description: "TestSubmission1-V1" +comment: TestComment1-V1 +additional_resources: +- {description: Test, location: test.ProSelecta, type: ProSelecta } +- {description: Test, location: test.tar.gz, type: HistFactory } +--- +name: "TestTable1-V1" +description: TestTable1-description-V1 +keywords: +- {name: cmenergies, values: [0]} +data_file: data1.yaml diff --git a/tests/test_data/test_analysis_submission/test.ProSelecta b/tests/test_data/test_analysis_submission/test.ProSelecta new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_data/test_analysis_submission/test.tar.gz b/tests/test_data/test_analysis_submission/test.tar.gz new file mode 100644 index 00000000..e69de29b