Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

data: rewrite provisionActivity field #663

Merged
merged 1 commit into from
Jan 6, 2020
Merged

data: rewrite provisionActivity field #663

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24,279 changes: 19,501 additions & 4,778 deletions data/documents_big.json
View file
Open in desktop

Large diffs are not rendered by default.

5,856 changes: 4,645 additions & 1,211 deletions data/documents_small.json
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions rero_ils/manual_translations.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ _('bf:Publication')
_('bf:Manufacture')
_('bf:Distribution')
_('bf:Production')
_('bf:Place')
_('Date')

# Harvested source
_('ebibliomedia')
Expand Down
59 changes: 40 additions & 19 deletions rero_ils/modules/documents/dojson/contrib/marc21tojson/model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -321,50 +321,51 @@ def marc21_to_provisionActivity(self, key, value):
"""
def build_statement(field_value, ind2):

def build_place_or_agent_data(code, label, index, link, add_country):
def build_agent_data(code, label, index, link):
type_per_code = {
'a': 'bf:Place',
'b': 'bf:Agent'
}
place_or_agent_data = {
agent_data = {
'type': type_per_code[code],
'label': [{'value': remove_trailing_punctuation(label)}]
}

if add_country:
if marc21tojson.cantons:
place_or_agent_data['canton'] = marc21tojson.cantons
if marc21tojson.country:
place_or_agent_data['country'] = marc21tojson.country
try:
alt_gr = marc21tojson.alternate_graphic['264'][link]
subfield = \
marc21tojson.get_subfields(alt_gr['field'])[index]
place_or_agent_data['label'].append({
agent_data['label'].append({
'value': remove_trailing_punctuation(subfield),
'language': get_language_script(alt_gr['script'])
})
except Exception as err:
pass
return place_or_agent_data
return agent_data

# function build_statement start here
tag_link, link = get_field_link_data(field_value)
items = get_field_items(field_value)
statement = []
index = 1
add_country = ind2 in (' ', '1')
for blob_key, blob_value in items:
if blob_key in ('a', 'b'):
place_or_agent_data = build_place_or_agent_data(
blob_key, blob_value, index, link, add_country)
if blob_key == 'a':
add_country = False
statement.append(place_or_agent_data)
agent_data = build_agent_data(
blob_key, blob_value, index, link)
statement.append(agent_data)
if blob_key != '__order__':
index += 1
return statement

def build_place():
place = {}
if marc21tojson.cantons:
place['canton'] = marc21tojson.cantons[0]
if marc21tojson.country:
place['country'] = marc21tojson.country
if place:
place['type'] = 'bf:Place'
return place

# the function marc21_to_provisionActivity start here
ind2 = key[4]
type_per_ind2 = {
Expand All @@ -380,9 +381,6 @@ def build_place_or_agent_data(code, label, index, link, add_country):
}

subfields_c = utils.force_list(value.get('c'))
if subfields_c:
subfield_c = subfields_c[0]
publication['date'] = subfield_c
if ind2 in (' ', '1'):
start_date = make_year(marc21tojson.date1_from_008)
if start_date:
Expand All @@ -393,7 +391,30 @@ def build_place_or_agent_data(code, label, index, link, add_country):
if (marc21tojson.date_type_from_008 == 'q' or
marc21tojson.date_type_from_008 == 'n'):
publication['note'] = 'Date(s) incertaine(s) ou inconnue(s)'
place = build_place()
if place:
publication['place'] = [place]
publication['statement'] = build_statement(value, ind2)
if subfields_c:
subfield_c = subfields_c[0]
date = {
'label': [{'value': subfield_c}],
'type': 'Date'
}

tag_link, link = get_field_link_data(value)
try:
alt_gr = marc21tojson.alternate_graphic['264'][link]
subfield = \
marc21tojson.get_subfields(alt_gr['field'], code='c')
date['label'].append({
'value': subfield[0],
'language': get_language_script(alt_gr['script'])
})
except Exception as err:
pass

publication['statement'].append(date)
return publication or None


Expand Down
68 changes: 39 additions & 29 deletions rero_ils/modules/documents/dojson/contrib/unimarctojson/model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,7 @@ def unimarc_to_edition_statement(self, key, value):
@utils.ignore_value
def unimarc_publishers_provision_activity_publication(self, key, value):
"""Get provision activity dates."""
def build_place_or_agent_data(code, label, index, add_country):
def build_place_or_agent_data(code, label, index):
type_per_code = {
'a': 'bf:Place',
'c': 'bf:Agent'
Expand All @@ -208,16 +208,20 @@ def build_place_or_agent_data(code, label, index, add_country):
'type': type_per_code[code],
'label': [{'value': remove_trailing_punctuation(label)}]
}
if add_country:
# country from 102
field_102 = unimarctojson.get_fields(tag='102')
if field_102:
field_102 = field_102[0]
country_codes = unimarctojson.get_subfields(field_102, 'a')
if country_codes:
place_or_agent_data['country'] = country_codes[0].lower()
return place_or_agent_data

def build_place():
# country from 102
place = {}
field_102 = unimarctojson.get_fields(tag='102')
if field_102:
field_102 = field_102[0]
country_codes = unimarctojson.get_subfields(field_102, 'a')
if country_codes:
place['country'] = country_codes[0].lower()
place['type'] = 'bf:Place'
return place

publication = {}
ind2 = key[4]
type_per_ind2 = {
Expand All @@ -243,22 +247,6 @@ def build_place_or_agent_data(code, label, index, add_country):
'type': type_per_ind2[ind2],
'statement': [],
}
subfields_d = utils.force_list(value.get('d'))
if subfields_d:
subfield_d = subfields_d[0]
publication['date'] = subfield_d
if ind2 in (' ', '_', '0'):
dates = subfield_d.replace('[', '').replace(']', '').split('-')
try:
if re.search(r'(^\[?\d{4}$)', dates[0]):
publication['startDate'] = dates[0]
except Exception:
pass
try:
if re.search(r'(^\d{4}\]?$)', dates[1]):
publication['endDate'] = dates[1]
except Exception:
pass

# TODO: dates from 100 not working !!!!
# if ind2 in (' ', '_', '1'):
Expand All @@ -280,18 +268,40 @@ def build_place_or_agent_data(code, label, index, add_country):
statement = []
items = get_field_items(value)
index = 1
add_country = ind2 in (' ', '_', '0')
for blob_key, blob_value in items:
if blob_key in ('a', 'c'):
place_or_agent_data = build_place_or_agent_data(
blob_key, blob_value, index, add_country)
if blob_key == 'a':
add_country = False
blob_key, blob_value, index)
statement.append(place_or_agent_data)
if blob_key != '__order__':
index += 1
if statement:
publication['statement'] = statement
if ind2 in (' ', '_', '0'):
place = build_place()
if place:
publication['place'] = [place]

subfields_d = utils.force_list(value.get('d'))
if subfields_d:
subfield_d = subfields_d[0]
publication['statement'].append({
'label': [{'value': subfield_d}],
'type': 'Date'
})
if ind2 in (' ', '_', '0'):
dates = subfield_d.replace('[', '').replace(']', '').split('-')
try:
if re.search(r'(^\[?\d{4}$)', dates[0]):
publication['startDate'] = dates[0]
except Exception:
pass
try:
if re.search(r'(^\d{4}\]?$)', dates[1]):
publication['endDate'] = dates[1]
except Exception:
pass

if not publication.get('statement'):
publication = None
return publication or None
Expand Down
54 changes: 32 additions & 22 deletions rero_ils/modules/documents/jsonschemas/documents/document-minimal-v0.0.1_src.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1171,9 +1171,36 @@
"bf:Production"
]
},
"place": {
"title": "Country and/or canton of the provision activity",
"type": "array",
"minItems": 1,
"items": {
"type": "object",
"required": [
"country"
],
"properties": {
"type": {
"const": "bf:Place"
},
"country": {
"title": "Country",
"type": "string",
"$ref": "#/definitions/country"
},
"canton": {
"title": "Canton",
"type": "string",
"$ref": "#/definitions/canton"
}
}
}
},
"statement": {
"title": "Statement of place and/or agent of the provision activity",
"title": "Statement of place and/or agent and/or date of the provision activity",
"type": "array",
"minItems": 1,
"items": {
"type": "object",
"properties": {
Expand All @@ -1182,31 +1209,19 @@
"type": "string",
"enum": [
"bf:Place",
"bf:Agent"
"bf:Agent",
"Date"
]
},
"canton": {
"title": "Canton",
"type": "array",
"minItems": 1,
"items": {
"type": "string",
"$ref": "#/definitions/canton"
}
},
"country": {
"title": "Country",
"type": "string",
"$ref": "#/definitions/country"
},
"label": {
"title": "Label",
"type": "array",
"minItems": 1,
"items": {
"type": "object",
"properties": {
"value": {
"title": "Place and/or agent name",
"title": "Place and/or agent name and/or date",
"type": "string",
"minLength": 1
},
Expand Down Expand Up @@ -1239,11 +1254,6 @@
"type": "string",
"minimum": -9999,
"maximum": 2050
},
"date": {
"title": "Date of publication (free formed)",
"description": "Date of the publication in a free form. If there's a normalized date of publication, then a free formed date can be added to be displayed.",
"type": "string"
}
}
}
Expand Down
54 changes: 32 additions & 22 deletions rero_ils/modules/documents/jsonschemas/documents/document-v0.0.1_src.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1169,9 +1169,36 @@
"bf:Production"
]
},
"place": {
rerowep marked this conversation as resolved.
Show resolved Hide resolved
"title": "Country and/or canton of the provision activity",
"type": "array",
"minItems": 1,
"items": {
"type": "object",
"required": [
"country"
],
"properties": {
"type": {
"const": "bf:Place"
},
"country": {
"title": "Country",
"type": "string",
"$ref": "#/definitions/country"
},
"canton": {
"title": "Canton",
"type": "string",
"$ref": "#/definitions/canton"
}
}
}
},
"statement": {
"title": "Statement of place and/or agent of the provision activity",
"title": "Statement of place and/or agent and/or date of the provision activity",
"type": "array",
"minItems": 1,
"items": {
"type": "object",
"properties": {
Expand All @@ -1180,31 +1207,19 @@
"type": "string",
"enum": [
"bf:Place",
"bf:Agent"
"bf:Agent",
"Date"
]
},
"canton": {
"title": "Canton",
"type": "array",
"minItems": 1,
"items": {
"type": "string",
"$ref": "#/definitions/canton"
}
},
"country": {
"title": "Country",
"type": "string",
"$ref": "#/definitions/country"
},
"label": {
"title": "Label",
"type": "array",
"minItems": 1,
"items": {
"type": "object",
"properties": {
"value": {
"title": "Place and/or agent name",
"title": "Place and/or agent name and/or date",
"type": "string",
"minLength": 1
},
Expand Down Expand Up @@ -1237,11 +1252,6 @@
"type": "string",
"minimum": -9999,
"maximum": 2050
},
"date": {
"title": "Date of publication (free formed)",
"description": "Date of the publication in a free form. If there's a normalized date of publication, then a free formed date can be added to be displayed.",
"type": "string"
}
}
}
Expand Down
Loading