Merge branch 'release-v33.2.0'

lib/create_mapper.py

@@ -28,10 +28,6 @@ def _create_scdl_mapper(data):
         from dplaingestion.mappers.scdl_mapper import SCDLMapper
         return SCDLMapper(data)
 
-    def _create_charleston_mapper(data):
-        from dplaingestion.mappers.charleston_mapper import SCDLCharlestonMapper
-        return SCDLCharlestonMapper(data)
-
     def _create_edan_mapper(data):
         from dplaingestion.mappers.edan_mapper import EDANMapper
         return EDANMapper(data)
@@ -91,7 +87,6 @@ def _create_missouri_mapper(data):
         'mdl':          lambda d: _create_mdl_mapper(d),
         'gpo':          lambda d: _create_gpo_mapper(d),
         'scdl':         lambda d: _create_scdl_mapper(d),
-        'charleston':   lambda d: _create_charleston_mapper(d),
         'edan':         lambda d: _create_edan_mapper(d),
         'nara':         lambda d: _create_nara_mapper(d),
         'nypl':         lambda d: _create_nypl_mapper(d),

lib/mappers/gpo_mapper.py

@@ -232,10 +232,6 @@ def update_date(self):
         if date:
             self.update_source_resource({"date": date})
 
-    def update_data_provider(self):
-        data_provider = self._get_mapped_value("provider/name")
-        self.mapped_data.update({"dataProvider": data_provider})
-
     def update_description(self):
         description = []
         if (not self.description["310"] and self.leader[7] == "s" and
@@ -276,7 +272,6 @@ def update_mapped_fields(self):
         self.update_date()
         self.update_description()
         self.update_rights()
-        self.update_data_provider()
         self.update_object()
 
     def map(self):

lib/mappers/ia_mapper.py

@@ -1,3 +1,4 @@
+from itertools import cycle
 from dplaingestion.utilities import iterify
 from dplaingestion.selector import exists, getprop
 from dplaingestion.mappers.mapper import Mapper
@@ -14,20 +15,63 @@ def __init__(self, provider_data):
         super(IAMapper, self).__init__(provider_data)
         self.meta_key = "metadata/"
 
-    def _map_meta(self, to_prop, from_props=None, source_resource=True):
+    def _add_to_list(self, lst, item):
+        """Allows a list to be appended or extended conditionally in place.
+
+        In Python, the `list.extend()` method can be used to combine two
+        lists. However, if `list.extend()` is used to extend a list with 
+        a string, the string will be iterated over as if it were a list. 
+        For example, consider the following:
+
+        >>> lst = ["1", "2", "3"]
+        >>> lst.extend("abc")
+        >>> lst
+        ["1", "2", "3", "a", "b", "c"]
+
+        When working with strings, `list.append()` uses the desired behavior.
+        """
+        if isinstance(item, list):
+            lst.extend(item)
+        else:
+            lst.append(item)
+
+    def _generate_map_dict(self, to_prop, from_props=None,
+                           prevent_single_item_lists=True):
+        """Generates a dict used for mapping metadata.
+
+        to_prop specifies the name of the key in the resulting dict.
+        from_props specifies the name of the key(s) in the incoming data.
+        If from_props is None, it is set to the value of to_prop.
+        prevent_single_item_lists will modify prop_values (the returned 
+        property values from from_props) to return a single string if it is
+        a single item list.
+        """
         if from_props is None:
             from_props = [to_prop]
 
-        prop_value = []
+        prop_values = []
         for prop in iterify(from_props):
             from_prop = self.meta_key + prop
             if exists(self.provider_data, from_prop):
-                prop_value.append(getprop(self.provider_data, from_prop))
+                self._add_to_list(prop_values, getprop(self.provider_data,
+                                                       from_prop))
+        if len(prop_values) == 1 and prevent_single_item_lists:
+            prop_values = prop_values[0]
+        if prop_values:
+            return {to_prop: prop_values}
+        else:
+            return {}
 
-        if len(prop_value) == 1:
-            prop_value = prop_value[0]
-        if prop_value:
-            _dict = {to_prop: prop_value}
+    def _map_meta(self, to_prop, from_props=None, source_resource=True):
+        """Helper function to update the `mapped_data` in a IAMapper object.
+
+        to_prop and from_props are described in IAMapper_generate_map_dict().
+        source_resource indicates whether to_prop should be updated in the
+        context of source_resource (if True) or the top-level object (if
+        False). 
+        """
+        _dict = self._generate_map_dict(to_prop, from_props)
+        if _dict:
             if source_resource:
                 self.update_source_resource(_dict)
             else:
@@ -61,7 +105,21 @@ def map_identifier(self):
         self._map_meta("identifier", ["identifier", "call_number"])
 
     def map_title(self):
-        self._map_meta("title", ["title", "volume"])
+        ia_titles = self._generate_map_dict("title", from_props=None,
+                                            prevent_single_item_lists=False)
+        volumes = self._generate_map_dict("volume", from_props=None,
+                                            prevent_single_item_lists=False)
+        print volumes
+        if volumes:
+            if len(ia_titles.get("title")) > len(volumes.get("volume")):
+                titles = [", ".join(t) for t in zip(ia_titles["title"],
+                                                    cycle(volumes["volume"]))]
+            else:
+                titles = [", ".join(t) for t in zip(cycle(ia_titles["title"]),
+                                                    volumes["volume"])]
+            self.update_source_resource({"title": titles})
+        else:
+            self.update_source_resource(ia_titles)
 
     def map_data_provider(self):
         self._map_meta("dataProvider", "contributor", False)

profiles/artstor.pjs

@@ -77,6 +77,7 @@
         "/set_type_from_physical_format",
         "/copy_prop?prop=sourceResource%2Fpublisher&to_prop=dataProvider",
         "/unset_prop?prop=sourceResource%2Frelation",
+        "/copy_prop?prop=provider%2Fname&to_prop=dataProvider&skip_if_exists=True",
         "/validate_mapv3"
     ],
     "thresholds": {                                                             

profiles/bpl.pjs

@@ -33,6 +33,7 @@
         "/set_prop?prop=provider%2Fname&value=Digital%20Commonwealth",
         "/set_prop?prop=sourceResource%2FstateLocatedIn&value=Massachusetts",
         "/enrich_location?prop=sourceResource%2FstateLocatedIn",
+        "/copy_prop?prop=provider%2Fname&to_prop=dataProvider&skip_if_exists=True",
         "/unset_prop?prop=_id&condition_prop=sourceResource%2Ftitle&condition=finding_aid_title",
         "/validate_mapv3"
     ],

profiles/digitalnc.pjs

@@ -28,6 +28,7 @@
         "/enrich_language",
         "/set_prop?prop=sourceResource%2FstateLocatedIn&value=North%20Carolina",
         "/enrich_location?prop=sourceResource%2FstateLocatedIn",
+        "/copy_prop?prop=provider%2Fname&to_prop=dataProvider&skip_if_exists=True",
         "/validate_mapv3"
     ],
     "thresholds": {                                                             

profiles/getty.pjs

@@ -32,6 +32,7 @@
         "/enrich-type",
         "/enrich_language",
         "/set_type_from_physical_format",
+        "/copy_prop?prop=provider%2Fname&to_prop=dataProvider&skip_if_exists=True",
         "/validate_mapv3"
     ],
     "thresholds": {

profiles/gpo.pjs

@@ -27,6 +27,7 @@
         "/enrich_location",
         "/geocode",
         "/enrich_language",
+        "/copy_prop?prop=provider%2Fname&to_prop=dataProvider&skip_if_exists=True",
         "/validate_mapv3"
     ],
     "thresholds": {

profiles/hathi.pjs

@@ -29,6 +29,7 @@
         "/unset_prop?prop=_id&condition=hathi_exclude&condition_prop=dataProvider",
         "/hathi_identify_object",
         "/set_prop?prop=provider&_dict=True&value=%7B%22%40id%22%3A%22http%3A%2F%2Fdp.la%2Fapi%2Fcontributor%2Fhathitrust%22%2C%22name%22%3A%22HathiTrust%22%7D",
+        "/copy_prop?prop=provider%2Fname&to_prop=dataProvider&skip_if_exists=True",
         "/validate_mapv3"
     ],
     "set_provider": [

profiles/ia.pjs

@@ -35,6 +35,9 @@
         },
         "MontanaStateLibrary": {
             "title": "Montana State Library"
+        },
+        "yivoinstitutelibrary": {
+            "title": "YIVO Institute Library"
         }
     },
     "contributor": {

profiles/kentucky.pjs

@@ -38,6 +38,7 @@
         "/unset_prop?prop=dataProvider",
         "/copy_prop?prop=sourceResource%2Fpublisher&to_prop=dataProvider",
         "/unset_prop?prop=sourceResource%2Fpublisher",
+        "/copy_prop?prop=provider%2Fname&to_prop=dataProvider&skip_if_exists=True",
         "/geocode",
         "/validate_mapv3"
     ],

profiles/minnesota.pjs

@@ -2,7 +2,7 @@
     "name": "minnesota", 
     "type": "oai_verbs",
     "metadata_prefix": "qdc",
-    "endpoint_url": "http://reflections.mndigital.org/oai/oai2.php",
+    "endpoint_url": "http://reflections.mndigital.org/oai/oai.php",
     "contributor": {
         "@id": "http://dp.la/api/contributor/mdl", 
         "name": "Minnesota Digital Library"
@@ -36,6 +36,7 @@
         "/enrich_language",
         "/mdl_state_located_in",
         "/enrich_location?prop=sourceResource%2FstateLocatedIn",
+        "/copy_prop?prop=provider%2Fname&to_prop=dataProvider&skip_if_exists=True",
         "/validate_mapv3"
     ],
     "thresholds": {

profiles/nypl.pjs

@@ -36,6 +36,7 @@
         "/geocode",
         "/enrich_language",
         "/enrich_location?prop=sourceResource%2FstateLocatedIn",
+        "/copy_prop?prop=provider%2Fname&to_prop=dataProvider&skip_if_exists=True",
         "/validate_mapv3"
     ],
     "thresholds": {

profiles/texas.pjs

@@ -332,6 +332,7 @@
         "/texas_enrich_location",
         "/geocode",
         "/enrich_language",
+        "/copy_prop?prop=provider%2Fname&to_prop=dataProvider&skip_if_exists=True",
         "/validate_mapv3"
     ],
     "thresholds": {

profiles/uiuc_book.pjs

@@ -31,6 +31,7 @@
         "/enrich_location",
         "/geocode",
         "/enrich_language",
+        "/copy_prop?prop=provider%2Fname&to_prop=dataProvider&skip_if_exists=True",
         "/validate_mapv3"
     ],
     "thresholds": {

profiles/usc.pjs

@@ -77,6 +77,7 @@
         "/remove_list_values?prop=sourceResource%2Ftype&values=item",
         "/dedup_value?prop=sourceResource%2Ftype",
         "/unset_prop?prop=sourceResource%2Fcollection",
+        "/copy_prop?prop=provider%2Fname&to_prop=dataProvider&skip_if_exists=True",
         "/validate_mapv3"
     ],
     "thresholds": {

setup.py

@@ -31,7 +31,7 @@
 from distutils.core import setup
 
 setup( name = 'ingestion',
-       version = '33.1.6',
+       version = '33.2.0',
        description='DPLA Ingestion System',
        author='Digital Public Library of America',
        author_email='tech@dp.la',