Associated Works for Zenodo Records

ADSCitationCapture/api.py

@@ -167,7 +167,7 @@ def get_github_metadata(app, citation_url):
         github_api = None
         try:
             path = urllib.parse.urlparse(citation_url).path.split("/")
-            github_api = app.conf['GITHUB_API_URL']+"repos/{}/{}/license".format(path[1],path[2])
+            github_api = app.conf['GITHUB_API_URL']+"repos/{}/{}/license".format(path[1],path[2]) if path[1] else None
 
         except Exception as e:
             msg = "Failed to parse :{} with Exception: {}".format(citation_url,e)

ADSCitationCapture/db.py

@@ -37,7 +37,7 @@ def store_event(app, data):
             stored = True
     return stored
 
-def store_citation_target(app, citation_change, content_type, raw_metadata, parsed_metadata, status):
+def store_citation_target(app, citation_change, content_type, raw_metadata, parsed_metadata, status, associated=None):
     """
     Stores a new citation target in the DB
     """
@@ -51,6 +51,7 @@ def store_citation_target(app, citation_change, content_type, raw_metadata, pars
         citation_target.curated_metadata = {}
         citation_target.status = status
         citation_target.bibcode = parsed_metadata.get("bibcode", None)
+        citation_target.associated_works = associated
         session.add(citation_target)
         try:
             session.commit()
@@ -62,7 +63,7 @@ def store_citation_target(app, citation_change, content_type, raw_metadata, pars
             stored = True
     return stored
 
-def _update_citation_target_metadata_session(session, content, raw_metadata, parsed_metadata, curated_metadata = {}, status=None, bibcode = None):
+def _update_citation_target_metadata_session(session, content, raw_metadata, parsed_metadata, curated_metadata = {}, status=None, bibcode=None, associated=None):
     """
     Actual calls to database session for update_citation_target_metadata
     """
@@ -79,6 +80,9 @@ def _update_citation_target_metadata_session(session, content, raw_metadata, par
         citation_target.parsed_cited_metadata = parsed_metadata
         citation_target.curated_metadata = curated_metadata
         citation_target.bibcode = bibcode
+        if(citation_target.associated_works != associated):
+                logger.debug("associated works set for {} set from {} to {}".format(citation_target.content, citation_target.associated_works, associated))
+                citation_target.associated_works = associated
         if status is not None:
             citation_target.status = status
         session.add(citation_target)
@@ -87,14 +91,14 @@ def _update_citation_target_metadata_session(session, content, raw_metadata, par
         metadata_updated = True
         return metadata_updated
 
-def update_citation_target_metadata(app, content, raw_metadata, parsed_metadata, curated_metadata = {}, status=None, bibcode = None):
+def update_citation_target_metadata(app, content, raw_metadata, parsed_metadata, curated_metadata={}, status=None, bibcode=None, associated=None):
     """
     Update metadata for a citation target
     """
     metadata_updated = False
     if not bibcode: bibcode = parsed_metadata.get('bibcode', None)
     with app.session_scope() as session:
-        metadata_updated =  _update_citation_target_metadata_session(session, content, raw_metadata, parsed_metadata, curated_metadata, status, bibcode)
+        metadata_updated =  _update_citation_target_metadata_session(session, content, raw_metadata, parsed_metadata, curated_metadata, status=status, bibcode=bibcode, associated=associated)
     return metadata_updated
 
 
@@ -148,9 +152,11 @@ def _extract_key_citation_target_data(records_db, disable_filter=False):
         {
             'bibcode': record_db.bibcode,
             'alternate_bibcode': record_db.parsed_cited_metadata.get('alternate_bibcode', []),
+            'version': record_db.parsed_cited_metadata.get('version', None),
             'content': record_db.content,
             'content_type': record_db.content_type,
             'curated_metadata': record_db.curated_metadata if record_db.curated_metadata is not None else {},
+            'associated_works': record_db.associated_works,
         }
         for record_db in records_db
         if disable_filter or record_db.parsed_cited_metadata.get('bibcode', None) is not None
@@ -200,13 +206,31 @@ def _get_citation_targets_session(session, only_status='REGISTERED'):
     """
     if only_status:
         records_db = session.query(CitationTarget).filter_by(status=only_status).all()
-        disable_filter = only_status in ['DISCARDED','EMITTABLE']
+        disable_filter = only_status in ['DISCARDED', 'EMITTABLE']
     else:
         records_db = session.query(CitationTarget).all()
         disable_filter = True
     records = _extract_key_citation_target_data(records_db, disable_filter=disable_filter)
     return records
-
+
+def get_associated_works_by_doi(app, all_versions_doi, only_status='REGISTERED'):
+    dois = all_versions_doi['versions']
+    concept_doi = all_versions_doi['concept_doi'].lower()
+    try:
+        versions = {"Version "+str(records.get('version', '')): records.get('bibcode', '') for records in get_citation_targets_by_doi(app, dois, only_status)}
+        root_ver = get_citation_targets_by_doi(app, [concept_doi], only_status)
+        if root_ver != []:
+            root_record = {'Software Source':root_ver[0]['bibcode']}
+            versions.update(root_record)
+        if versions != {}:
+            return versions
+        else:
+            logger.info('No associated works for %s in database', dois[0])
+            return None
+    except:
+        logger.info('No associated works for %s in database', dois[0])
+        return None
+
 def get_citation_targets(app, only_status='REGISTERED'):
     """
     Return a list of dict with all citation targets (or only the registered ones)
@@ -382,13 +406,14 @@ def populate_bibcode_column(main_session):
     records = _get_citation_targets_session(main_session, only_status = None)
     for record in records:
         content = record.get('content', None)
-        bibcode = record
+        bibcode = record.get('bibcode', None)
+        associated = record.get('associate_works', {})
         logger.debug("Collecting metadata for {}".format(record.get('content')))
         citation_in_db = False
         metadata = {}
         metadata = _get_citation_target_metadata_session(main_session, content, citation_in_db, metadata, curate=False)
         if metadata:
-            logger.debug("Updating Bibcode field for {}".format(record.get('content')))
+            logger.debug("Populating Bibcode field for {}".format(record.get('content')))
             raw_metadata = metadata.get('raw', {})
             parsed_metadata = metadata.get('parsed', {})
             curated_metadata = metadata.get('curated',{})
@@ -402,5 +427,5 @@ def populate_bibcode_column(main_session):
             else:
                 bibcode = parsed_metadata.get('bibcode',None)
 
-            _update_citation_target_metadata_session(main_session, content, raw_metadata, parsed_metadata, curated_metadata, status, bibcode)
+            _update_citation_target_metadata_session(main_session, content, raw_metadata, parsed_metadata, curated_metadata, status, bibcode, associated)
 

ADSCitationCapture/doi.py

@@ -209,3 +209,46 @@ def _parse_metadata_zenodo_doi(raw_metadata):
             parsed_metadata['bibcode'] = bibcode
     return parsed_metadata
 
+def fetch_all_versions_doi(base_doi_url, base_datacite_url, parsed_metadata):  
+    """
+    Takes zenodo parsed metadata and fetches DOI for all versions of zenodo repository
+    """
+    return _fetch_all_versions_doi(base_doi_url, base_datacite_url, parsed_metadata)
+
+def _fetch_all_versions_doi(base_doi_url, base_datacite_url, parsed_metadata):
+    """
+    Takes zenodo parsed metadata and fetches DOI for base repository as well as DOI for all versions.
+    """
+
+    if parsed_metadata.get('version_of', None) not in (None,"",[],''):
+    #check if target is a software version and not the base doi.
+        try:
+            logger.info("{} is version of: {}".format(parsed_metadata['bibcode'], parsed_metadata.get('version_of', None)))
+            #try to recover the base doi for the target
+            raw_metadata = fetch_metadata(base_doi_url, base_datacite_url, parsed_metadata.get('version_of')[0])
+            parsed_all_version = parse_metadata(raw_metadata)
+            if parsed_all_version is not None:
+                logger.debug("Found Associated Versions: {}".format(parsed_all_version.get('versions', None)))
+            #return dois for all versions of the target software and the base doi.
+            versions_json = {'concept_doi': parsed_metadata.get('version_of', None)[0], 'versions': parsed_all_version.get('versions', None)}
+            logger.debug("{} version dict is {}".format(parsed_metadata['bibcode'], parsed_all_version.get('versions', None)))
+            return versions_json
+        except Exception as e:
+            logger.exception("Failed to fetch metadata with Exception: {}".format(e))
+            return {'concept_doi': None, 'versions': None}
+
+    elif parsed_metadata.get('versions',None) not in (None, [],""):
+        #If citation target is base doi for software.
+        logger.info("{} is the root version".format(parsed_metadata["properties"]['DOI']))
+
+        try:
+            #return all versions including the base doi.
+            logger.debug("Found Associated Versions: {}".format(parsed_metadata.get('versions',None)))
+            return {'concept_doi': parsed_metadata.get('properties')['DOI'], 'versions': parsed_metadata.get('versions',None)}
+
+        except Exception as e:
+            logger.exception("Attempt to return versions failed with Exception: {}".format(e))
+            return {'concept_doi': None, 'versions': None}
+
+    else:
+        return {'concept_doi': None, 'versions': None}

ADSCitationCapture/forward.py

@@ -22,7 +22,7 @@
 
 
 # =============================== FUNCTIONS ======================================= #
-def build_record(app, citation_change, parsed_metadata, citations, entry_date=None):
+def build_record(app, citation_change, parsed_metadata, citations, db_versions, entry_date=None):
     if citation_change.content_type != CitationChangeContentType.doi:
         raise Exception("Only DOI records can be forwarded to master")
     # Extract required values
@@ -125,11 +125,11 @@ def build_record(app, citation_change, parsed_metadata, citations, entry_date=No
     else:
         status = 0 # active
     record = DenormalizedRecord(**record_dict)
-    nonbib_record = _build_nonbib_record(app, citation_change, record, status)
+    nonbib_record = _build_nonbib_record(app, citation_change, record, db_versions, status)
     return record, nonbib_record
 
 
-def _build_nonbib_record(app, citation_change, record, status):
+def _build_nonbib_record(app, citation_change, record, db_versions, status):
     doi = citation_change.content
     nonbib_record_dict = {
         'status': status,
@@ -139,8 +139,8 @@ def _build_nonbib_record(app, citation_change, record, status):
         'data': [],
         'data_links_rows': [
             {'link_type': 'ESOURCE', 'link_sub_type': 'PUB_HTML',
-                     'url': [app.conf['DOI_URL'] + doi], 'title': [''], 'item_count':0}, # `item_count` only used for DATA and not ESOURCES
-        ],
+                     'url': [app.conf['DOI_URL'] + doi], 'title': [''], 'item_count':0},
+                     ], # `item_count` only used for DATA and not ESOURCES
         'citation_count_norm': record.citation_count_norm,
         'grants': [],
         'ned_objects': [],
@@ -150,6 +150,9 @@ def _build_nonbib_record(app, citation_change, record, status):
         'simbad_objects': [],
         'total_link_counts': 0 # Only used for DATA and not for ESOURCES
     }
+    if db_versions not in [{"":""}, None]:
+        nonbib_record_dict['data_links_rows'].append({'link_type': 'ASSOCIATED', 'link_sub_type': '', 
+                     'url': db_versions.values(), 'title': db_versions.keys(), 'item_count':0})
     nonbib_record = NonBibRecord(**nonbib_record_dict)
     nonbib_record.esource.extend(record.esources)
     nonbib_record.reference.extend(record.reference)

ADSCitationCapture/models.py

@@ -74,6 +74,7 @@ class CitationTarget(Base):
     parsed_cited_metadata = Column(JSONB)
     curated_metadata = Column(JSONB)
     status = Column(target_status_type)
+    associated_works = Column(JSONB)
     created = Column(UTCDateTime, default=get_date)
     updated = Column(UTCDateTime, onupdate=get_date)
     citations = relationship("Citation", primaryjoin="CitationTarget.content==Citation.content")