persons: filter persons view

* Adds `organisations` to persons ES index.
* Improves persons (authors) indexing during document indexing.

Co-Authored-by: Peter Weber <peter.weber@rero.ch>

rero_ils/config.py

@@ -45,7 +45,7 @@
 from rero_ils.modules.api import IlsRecordIndexer
 
 from .modules.circ_policies.api import CircPolicy
-from .modules.documents.api import Document
+from .modules.documents.api import Document, DocumentsIndexer
 from .modules.holdings.api import Holding, HoldingsIndexer
 from .modules.item_types.api import ItemType
 from .modules.items.api import Item, ItemsIndexer
@@ -341,7 +341,7 @@ def _(x):
         search_class=RecordsSearch,
         search_index='documents',
         search_type=None,
-        indexer_class=IlsRecordIndexer,
+        indexer_class=DocumentsIndexer,
         record_serializers={
             'application/json': (
                 'rero_ils.modules.serializers:json_v1_response'

rero_ils/modules/documents/api.py

@@ -27,7 +27,7 @@
 from .models import DocumentIdentifier
 from .utils import edition_format_text, publication_statement_text, \
     series_format_text
-from ..api import IlsRecord
+from ..api import IlsRecord, IlsRecordIndexer
 from ..fetchers import id_fetcher
 from ..minters import id_minter
 from ..organisations.api import Organisation
@@ -45,6 +45,22 @@
 document_id_fetcher = partial(id_fetcher, provider=DocumentProvider)
 
 
+class DocumentsIndexer(IlsRecordIndexer):
+    """Indexing documents in Elasticsearch."""
+
+    def index(self, record):
+        """Index an document."""
+        return_value = super(DocumentsIndexer, self).index(record)
+        from ..persons.api import Person
+        for author in record.replace_refs().get('authors', []):
+            auth_pid = author.get('pid')
+            if auth_pid:
+                person = Person.get_record_by_pid(auth_pid)
+                if person:
+                    person.reindex()
+        return return_value
+
+
 class DocumentsSearch(RecordsSearch):
     """DocumentsSearch."""
 
@@ -60,6 +76,7 @@ class Document(IlsRecord):
     minter = document_id_minter
     fetcher = document_id_fetcher
     provider = DocumentProvider
+    indexer = DocumentsIndexer
 
     def is_available(self, view_code):
         """Get availability for document."""

rero_ils/modules/ext.py

@@ -36,6 +36,7 @@
 from .locations.listener import enrich_location_data
 from .notifications.listener import enrich_notification_data
 from .patrons.listener import enrich_patron_data
+from .persons.listener import enrich_persons_data
 from .persons.receivers import publish_api_harvested_records
 from ..filter import format_date_filter, jsondumps, text_to_id, to_pretty_json
 
@@ -86,6 +87,7 @@ def register_signals(self):
         """Register signals."""
         before_record_index.connect(enrich_loan_data)
         before_record_index.connect(enrich_document_data)
+        before_record_index.connect(enrich_persons_data)
         before_record_index.connect(enrich_item_data)
         before_record_index.connect(enrich_patron_data)
         before_record_index.connect(enrich_location_data)

rero_ils/modules/items/cli.py

@@ -102,7 +102,7 @@ def generate(count, itemscount, missing):
         workshop_item = 1
         documents_pids = Document.get_all_pids()
         with click.progressbar(
-                reversed(documents_pids[:count]), length=count) as bar:
+                reversed(list(documents_pids)[:count]), length=count) as bar:
             for document_pid in bar:
                 holdings = [{}]
                 if Document.get_record_by_pid(

rero_ils/modules/loans/cli.py

@@ -283,23 +283,24 @@ def get_loanable_items(patron_type_pid):
         .filter('term', status=ItemStatus.ON_SHELF).source(['pid']).scan()
     for loanable_item in loanable_items:
         item = Item.get_record_by_pid(loanable_item.pid)
-        circ_policy = CircPolicy.provide_circ_policy(
-            item.holding_library_pid,
-            patron_type_pid,
-            item.holding_circulation_category_pid
-        )
-        if (
-                circ_policy.get('allow_checkout') and
-                circ_policy.get('allow_requests')
-        ):
-            if not item.number_of_requests():
-                # exclude the first 16 items of the 3rd organisation
-                barcode = item.get('barcode')
-                if not (
-                    barcode.startswith('fictive') and
-                    int(barcode.split('fictive')[1]) < 17
-                ):
-                    yield item
+        if item:
+            circ_policy = CircPolicy.provide_circ_policy(
+                item.holding_library_pid,
+                patron_type_pid,
+                item.holding_circulation_category_pid
+            )
+            if (
+                    circ_policy.get('allow_checkout') and
+                    circ_policy.get('allow_requests')
+            ):
+                if not item.number_of_requests():
+                    # exclude the first 16 items of the 3rd organisation
+                    barcode = item.get('barcode')
+                    if not (
+                        barcode.startswith('fictive') and
+                        int(barcode.split('fictive')[1]) < 17
+                    ):
+                        yield item
 
 
 def get_random_pickup_location(patron_pid):

rero_ils/modules/persons/api.py

@@ -19,13 +19,15 @@
 
 from functools import partial
 
+from elasticsearch_dsl import A
 from flask import current_app
 from invenio_search.api import RecordsSearch
 from requests import codes as requests_codes
 from requests import get as requests_get
 
 from .models import PersonIdentifier
 from ..api import IlsRecord
+from ..documents.api import DocumentsSearch
 from ..fetchers import id_fetcher
 from ..minters import id_minter
 from ..providers import Provider
@@ -71,10 +73,7 @@ def get_record_by_mef_pid(cls, pid):
         metadata = data.get('metadata')
         if '$schema' in metadata:
             del metadata['$schema']
-        rec = cls.create(
-            metadata,
-            dbcommit=True,
-            reindex=True)
+        rec = cls.create(metadata, dbcommit=True, reindex=False)
         return rec
 
     def dumps_for_document(self):
@@ -161,3 +160,17 @@ def _get_author_for_document(self):
         if variant_person:
             author['variant_name'] = unique_list(variant_person)
         return author
+
+    @property
+    def organisation_pids(self):
+        """Get organisations pids."""
+        organisations = set()
+        search = DocumentsSearch().filter('term', authors__pid=self.pid)
+        agg = A('terms',
+                field='holdings.organisation.organisation_pid', size=10)
+        search.aggs.bucket('organisation', agg)
+        results = search.execute()
+        for result in results.aggregations.organisation.buckets:
+            if result.doc_count:
+                organisations.add(result.key)
+        return list(organisations)

rero_ils/modules/persons/listener.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+#
+# RERO ILS
+# Copyright (C) 2019 RERO
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""Signals connector for Persons."""
+
+from .api import PersonsSearch
+
+
+def enrich_persons_data(sender, json=None, record=None, index=None,
+                        doc_type=None, **dummy_kwargs):
+    """Signal sent before a record is indexed.
+
+    :param json: The dumped record dictionary which can be modified.
+    :param record: The record being indexed.
+    :param index: The index in which the record will be indexed.
+    :param doc_type: The doc_type for the record.
+    """
+    if index == '-'.join([PersonsSearch.Meta.index, doc_type]):
+        json['organisations'] = record.organisation_pids

rero_ils/modules/persons/mappings/v6/persons/person-v0.0.1.json

@@ -175,6 +175,9 @@
             }
           }
         },
+        "organisations": {
+          "type": "keyword"
+        },
         "_created": {
           "type": "date"
         },

scripts/setup

@@ -188,28 +188,28 @@ info_msg "Generate fixtures:"
 
 info_msg "- Organisations ${CREATE_LAZY} ${DONT_STOP}"
 eval ${PREFIX} pipenv run invenio fixtures create --pid_type org ${DATA_PATH}/organisations.json --append ${CREATE_LAZY} ${DONT_STOP}
-eval ${PREFIX} pipenv run invenio index reindex -t org --yes-i-know
+eval ${PREFIX} pipenv run invenio utils reindex -t org --yes-i-know
 
 info_msg "- Libraries: ${CREATE_LAZY} ${DONT_STOP}"
 eval ${PREFIX} pipenv run invenio fixtures create --pid_type lib ${DATA_PATH}/libraries.json --append ${CREATE_LAZY} ${DONT_STOP}
-eval ${PREFIX} pipenv run invenio index reindex -t lib --yes-i-know
+eval ${PREFIX} pipenv run invenio utils reindex -t lib --yes-i-know
 
 info_msg "- Locations: ${CREATE_LAZY} ${DONT_STOP}"
 eval ${PREFIX} pipenv run invenio fixtures create --pid_type loc ${DATA_PATH}/locations.json  --append ${CREATE_LAZY} ${DONT_STOP}
-eval ${PREFIX} pipenv run invenio index reindex -t loc --yes-i-know
+eval ${PREFIX} pipenv run invenio utils reindex -t loc --yes-i-know
 
 info_msg "- Item types: ${CREATE_LAZY} ${DONT_STOP}"
 eval ${PREFIX} pipenv run invenio fixtures create --pid_type itty ${DATA_PATH}/item_types.json  --append ${CREATE_LAZY} ${DONT_STOP}
-eval ${PREFIX} pipenv run invenio index reindex -t itty --yes-i-know
+eval ${PREFIX} pipenv run invenio utils reindex -t itty --yes-i-know
 
 info_msg "- Patron types: ${CREATE_LAZY} ${DONT_STOP}"
 eval ${PREFIX} pipenv run invenio fixtures create --pid_type ptty ${DATA_PATH}/patron_types.json --append ${CREATE_LAZY} ${DONT_STOP}
-eval ${PREFIX} pipenv run invenio index reindex -t ptty --yes-i-know
+eval ${PREFIX} pipenv run invenio utils reindex -t ptty --yes-i-know
 
 info_msg "- Circulation policies: ${CREATE_LAZY} ${DONT_STOP}"
 eval ${PREFIX} pipenv run invenio fixtures create --pid_type cipo ${DATA_PATH}/circulation_policies.json --append ${CREATE_LAZY} ${DONT_STOP}
-eval ${PREFIX} pipenv run invenio index reindex -t cipo --yes-i-know
-eval ${PREFIX} pipenv run invenio index run --raise-on-error
+eval ${PREFIX} pipenv run invenio utils reindex -t cipo --yes-i-know
+eval ${PREFIX} pipenv run invenio utils runindex --raise-on-error
 
 info_msg "- Users:"
 eval ${PREFIX} pipenv run invenio fixtures import_users ${DATA_PATH}/users.json -v
@@ -265,21 +265,21 @@ fi
 
 info_msg "- Holdings: ${HOLDINGS} ${CREATE_LAZY} ${DONT_STOP}"
 eval ${PREFIX} pipenv run invenio fixtures create --pid_type hold --schema 'http://ils.rero.ch/schema/holdings/holding-v0.0.1.json' ${HOLDINGS} --append ${CREATE_LAZY} ${DONT_STOP}
-eval ${PREFIX} pipenv run invenio index reindex -t hold --yes-i-know
-eval ${PREFIX} pipenv run invenio index run -c 4 --raise-on-error
+eval ${PREFIX} pipenv run invenio utils reindex -t hold --yes-i-know
+eval ${PREFIX} pipenv run invenio utils runindex -c 4 --raise-on-error
 
 info_msg "- Items: ${ITEMS} ${CREATE_LAZY} ${DONT_STOP}"
 eval ${PREFIX} pipenv run invenio fixtures create --pid_type item --schema 'http://ils.rero.ch/schema/items/item-v0.0.1.json' ${ITEMS} --append ${CREATE_LAZY} ${DONT_STOP}
 
 # index items
 info_msg "Index items"
-eval ${PREFIX} pipenv run invenio index reindex -t item --yes-i-know
-eval ${PREFIX} pipenv run invenio index run -c 4 --raise-on-error
+eval ${PREFIX} pipenv run invenio utils reindex -t item --yes-i-know
+eval ${PREFIX} pipenv run invenio utils runindex -c 4 --raise-on-error
 
 # index documents
 info_msg "Index Documents:"
-eval ${PREFIX} pipenv run invenio index reindex -t doc --yes-i-know
-eval ${PREFIX} pipenv run invenio index run -c 4 --raise-on-error
+eval ${PREFIX} pipenv run invenio utils reindex -t doc --yes-i-know
+eval ${PREFIX} pipenv run invenio utils runindex -c 4 --raise-on-error
 
 # create circulation transactions
 info_msg "Circulation transactions:"

tests/api/test_persons_rest.py

@@ -74,5 +74,7 @@ def test_persons_get(client, person):
     res = client.get(list_url)
     assert res.status_code == 200
     data = get_json(res)
+    person = person.replace_refs()
+    person['organisations'] = person.organisation_pids
 
     assert data['hits']['hits'][0]['metadata'] == person.replace_refs()

tests/ui/persons/test_persons_api.py

@@ -19,11 +19,10 @@
 
 from __future__ import absolute_import, print_function
 
-from rero_ils.modules.persons.api import Person, PersonsSearch, \
-    person_id_fetcher
+from rero_ils.modules.persons.api import Person, person_id_fetcher
 
 
-def test_person_create(db, person_data_tmp):
+def test_person_create(app, person_data_tmp):
     """Test MEF person creation."""
     pers = Person.get_record_by_pid('1')
     assert not pers
@@ -49,3 +48,5 @@ def test_person_create(db, person_data_tmp):
     )
     pers = Person.get_record_by_pid('2')
     assert pers.get('viaf_pid') == '1234'
+
+    assert pers.organisation_pids == []