Skip to content

Commit

Permalink
data: rewrite provisionActivity field
Browse files Browse the repository at this point in the history 
* Rewrites rero-ils, ebooks and unimarc dojson.
* Updates fixtures.
* Updates tests.
* Updates json schema.
* Updates json mapping.

Co-Authored-by: Peter Weber <peter.weber@rero.ch>
  • Loading branch information
@rerowep
rerowep and rerowep committed Dec 18, 2019
1 parent 0f22fe6 commit f957707
Show file tree
Hide file tree
Showing 17 changed files with 25,059 additions and 6,590 deletions.
24,279 changes: 19,501 additions & 4,778 deletions data/documents_big.json
View file

Large diffs are not rendered by default.

5,856 changes: 4,645 additions & 1,211 deletions data/documents_small.json
View file

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions rero_ils/manual_translations.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ _('bf:Publication')
_('bf:Manufacture')
_('bf:Distribution')
_('bf:Production')
_('bf:Place')
_('Date')

# Harvested source
_('ebibliomedia')
Expand Down
62 changes: 42 additions & 20 deletions rero_ils/modules/documents/dojson/contrib/marc21tojson/model.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@

import requests
from dojson import utils
from flask_babelex import gettext as _

from rero_ils.dojson.utils import ReroIlsMarc21Overdo, error_print, \
get_field_items, get_field_link_data, make_year, not_repetitive, \
Expand Down Expand Up @@ -321,50 +322,51 @@ def marc21_to_provisionActivity(self, key, value):
"""
def build_statement(field_value, ind2):

def build_place_or_agent_data(code, label, index, link, add_country):
def build_agent_data(code, label, index, link):
type_per_code = {
'a': 'bf:Place',
'b': 'bf:Agent'
}
place_or_agent_data = {
agent_data = {
'type': type_per_code[code],
'label': [{'value': remove_trailing_punctuation(label)}]
}

if add_country:
if marc21tojson.cantons:
place_or_agent_data['canton'] = marc21tojson.cantons
if marc21tojson.country:
place_or_agent_data['country'] = marc21tojson.country
try:
alt_gr = marc21tojson.alternate_graphic['264'][link]
subfield = \
marc21tojson.get_subfields(alt_gr['field'])[index]
place_or_agent_data['label'].append({
agent_data['label'].append({
'value': remove_trailing_punctuation(subfield),
'language': get_language_script(alt_gr['script'])
})
except Exception as err:
pass
return place_or_agent_data
return agent_data

# function build_statement start here
tag_link, link = get_field_link_data(field_value)
items = get_field_items(field_value)
statement = []
index = 1
add_country = ind2 in (' ', '1')
for blob_key, blob_value in items:
if blob_key in ('a', 'b'):
place_or_agent_data = build_place_or_agent_data(
blob_key, blob_value, index, link, add_country)
if blob_key == 'a':
add_country = False
statement.append(place_or_agent_data)
agent_data = build_agent_data(
blob_key, blob_value, index, link)
statement.append(agent_data)
if blob_key != '__order__':
index += 1
return statement

def build_place():
place = {}
if marc21tojson.cantons:
place['canton'] = marc21tojson.cantons[0]
if marc21tojson.country:
place['country'] = marc21tojson.country
if place:
place['type'] = 'bf:Place'
return place

# the function marc21_to_provisionActivity start here
ind2 = key[4]
type_per_ind2 = {
Expand All @@ -380,9 +382,6 @@ def build_place_or_agent_data(code, label, index, link, add_country):
}

subfields_c = utils.force_list(value.get('c'))
if subfields_c:
subfield_c = subfields_c[0]
publication['date'] = subfield_c
if ind2 in (' ', '1'):
start_date = make_year(marc21tojson.date1_from_008)
if start_date:
Expand All @@ -392,8 +391,31 @@ def build_place_or_agent_data(code, label, index, link, add_country):
publication['endDate'] = end_date
if (marc21tojson.date_type_from_008 == 'q' or
marc21tojson.date_type_from_008 == 'n'):
publication['note'] = 'Date(s) incertaine(s) ou inconnue(s)'
publication['note'] = _('Date(s) incertaine(s) ou inconnue(s)')
place = build_place()
if place:
publication['place'] = [place]
publication['statement'] = build_statement(value, ind2)
if subfields_c:
subfield_c = subfields_c[0]
date = {
'label': [{'value': subfield_c}],
'type': 'Date'
}

tag_link, link = get_field_link_data(value)
try:
alt_gr = marc21tojson.alternate_graphic['264'][link]
subfield = \
marc21tojson.get_subfields(alt_gr['field'], code='c')
date['label'].append({
'value': subfield[0],
'language': get_language_script(alt_gr['script'])
})
except Exception as err:
pass

publication['statement'].append(date)
return publication or None


Expand Down
68 changes: 39 additions & 29 deletions rero_ils/modules/documents/dojson/contrib/unimarctojson/model.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,7 @@ def unimarc_to_edition_statement(self, key, value):
@utils.ignore_value
def unimarc_publishers_provision_activity_publication(self, key, value):
"""Get provision activity dates."""
def build_place_or_agent_data(code, label, index, add_country):
def build_place_or_agent_data(code, label, index):
type_per_code = {
'a': 'bf:Place',
'c': 'bf:Agent'
Expand All @@ -208,16 +208,20 @@ def build_place_or_agent_data(code, label, index, add_country):
'type': type_per_code[code],
'label': [{'value': remove_trailing_punctuation(label)}]
}
if add_country:
# country from 102
field_102 = unimarctojson.get_fields(tag='102')
if field_102:
field_102 = field_102[0]
country_codes = unimarctojson.get_subfields(field_102, 'a')
if country_codes:
place_or_agent_data['country'] = country_codes[0].lower()
return place_or_agent_data

def build_place():
# country from 102
place = {}
field_102 = unimarctojson.get_fields(tag='102')
if field_102:
field_102 = field_102[0]
country_codes = unimarctojson.get_subfields(field_102, 'a')
if country_codes:
place['country'] = country_codes[0].lower()
place['type'] = 'bf:Place'
return place

publication = {}
ind2 = key[4]
type_per_ind2 = {
Expand All @@ -243,22 +247,6 @@ def build_place_or_agent_data(code, label, index, add_country):
'type': type_per_ind2[ind2],
'statement': [],
}
subfields_d = utils.force_list(value.get('d'))
if subfields_d:
subfield_d = subfields_d[0]
publication['date'] = subfield_d
if ind2 in (' ', '_', '0'):
dates = subfield_d.replace('[', '').replace(']', '').split('-')
try:
if re.search(r'(^\[?\d{4}$)', dates[0]):
publication['startDate'] = dates[0]
except Exception:
pass
try:
if re.search(r'(^\d{4}\]?$)', dates[1]):
publication['endDate'] = dates[1]
except Exception:
pass

# TODO: dates from 100 not working !!!!
# if ind2 in (' ', '_', '1'):
Expand All @@ -280,18 +268,40 @@ def build_place_or_agent_data(code, label, index, add_country):
statement = []
items = get_field_items(value)
index = 1
add_country = ind2 in (' ', '_', '0')
for blob_key, blob_value in items:
if blob_key in ('a', 'c'):
place_or_agent_data = build_place_or_agent_data(
blob_key, blob_value, index, add_country)
if blob_key == 'a':
add_country = False
blob_key, blob_value, index)
statement.append(place_or_agent_data)
if blob_key != '__order__':
index += 1
if statement:
publication['statement'] = statement
if ind2 in (' ', '_', '0'):
place = build_place()
if place:
publication['place'] = [place]

subfields_d = utils.force_list(value.get('d'))
if subfields_d:
subfield_d = subfields_d[0]
publication['statement'].append({
'label': [{'value': subfield_d}],
'type': 'Date'
})
if ind2 in (' ', '_', '0'):
dates = subfield_d.replace('[', '').replace(']', '').split('-')
try:
if re.search(r'(^\[?\d{4}$)', dates[0]):
publication['startDate'] = dates[0]
except Exception:
pass
try:
if re.search(r'(^\d{4}\]?$)', dates[1]):
publication['endDate'] = dates[1]
except Exception:
pass

if not publication.get('statement'):
publication = None
return publication or None
Expand Down
54 changes: 32 additions & 22 deletions rero_ils/modules/documents/jsonschemas/documents/document-minimal-v0.0.1_src.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1171,9 +1171,36 @@
"bf:Production"
]
},
"place": {
"title": "Country and/or canton of the provision activity",
"type": "array",
"minItems": 1,
"items": {
"type": "object",
"required": [
"country"
],
"properties": {
"type": {
"const": "bf:Place"
},
"country": {
"title": "Country",
"type": "string",
"$ref": "#/definitions/country"
},
"canton": {
"title": "Canton",
"type": "string",
"$ref": "#/definitions/canton"
}
}
}
},
"statement": {
"title": "Statement of place and/or agent of the provision activity",
"title": "Statement of place and/or agent and/or date of the provision activity",
"type": "array",
"minItems": 1,
"items": {
"type": "object",
"properties": {
Expand All @@ -1182,31 +1209,19 @@
"type": "string",
"enum": [
"bf:Place",
"bf:Agent"
"bf:Agent",
"Date"
]
},
"canton": {
"title": "Canton",
"type": "array",
"minItems": 1,
"items": {
"type": "string",
"$ref": "#/definitions/canton"
}
},
"country": {
"title": "Country",
"type": "string",
"$ref": "#/definitions/country"
},
"label": {
"title": "Label",
"type": "array",
"minItems": 1,
"items": {
"type": "object",
"properties": {
"value": {
"title": "Place and/or agent name",
"title": "Place and/or agent name and/or date",
"type": "string",
"minLength": 1
},
Expand Down Expand Up @@ -1239,11 +1254,6 @@
"type": "string",
"minimum": -9999,
"maximum": 2050
},
"date": {
"title": "Date of publication (free formed)",
"description": "Date of the publication in a free form. If there's a normalized date of publication, then a free formed date can be added to be displayed.",
"type": "string"
}
}
}
Expand Down
Loading

0 comments on commit f957707

Please sign in to comment.