operation logs: use an Elasticsearch only resource

* Renames virtua command CLI name. * Fixes monitor view to compute Elasticsearch and DB count diff when the index does not exists. * Creates an operation logs Elasticsearch record class. It creates one index per year. * Denies to all to read one record. * Adds a CLI to dumps the operation logs in a JSON file for backup. * Closes: #1725. Co-Authored-By: Johnny Mariéthoz <Johnny.Mariethoz@rero.ch>
rero · Jun 12, 2021 · 3323f70 · 3323f70
1 parent 9d9e7a8
commit 3323f70
Show file tree

Hide file tree

Showing 28 changed files with 502 additions and 476 deletions.
diff --git a/data/operation_logs.json b/data/operation_logs.json
@@ -1 +1,10 @@
-[]
+[
+  {
+    "record": {
+      "$ref": "https://ils.rero.ch/api/documents/1"
+    },
+    "operation": "update",
+    "user_name": "system",
+    "date": "2021-01-21T09:51:52.879533+00:00"
+  }
+]
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -40,7 +40,7 @@ invenio-oaiserver = ">=1.2.0,<1.3.0"
 invenio-pidstore = ">=1.2.1,<1.3.0"
 invenio-records-rest = ">=1.8.0,<1.9.0"
 invenio-records-ui= ">=1.2.0,<1.3.0"
-invenio-records = ">=1.4.0,<1.6.0"
+invenio-records = {version = ">=1.4.0,<1.6.0", allow-prereleases = true}
 
 ## Default from Invenio
 invenio = {version = ">=3.4.0,<5.4.0", extras = ["base", "postgresql", "auth", "elasticsearch7", "docs", "tests" ]}

diff --git a/rero_ils/config.py b/rero_ils/config.py
@@ -1613,13 +1613,12 @@ def _(x):
             action='delete', record=record, cls=TemplatePermission)
     ),
     oplg=dict(
+        # TODO: useless, but required
         pid_type='oplg',
-        pid_minter='operation_log_id',
+        pid_minter='recid',
         pid_fetcher='operation_log_id',
-        search_class='rero_ils.modules.operation_logs.api:OperationLogsSearch',
         search_index='operation_logs',
         search_type=None,
-        indexer_class='rero_ils.modules.operation_logs.api:OperationLogsIndexer',
         record_serializers={
             'application/json': (
                 'rero_ils.modules.serializers:json_v1_response'
@@ -1638,6 +1637,7 @@ def _(x):
         },
         record_class='rero_ils.modules.operation_logs.api:OperationLog',
         list_route='/operation_logs/',
+        # TODO: create a converter for es id, not used for the moment.
         item_route='/operation_logs/<pid(oplg, record_class='
         '"rero_ils.modules.operation_logs.api:OperationLog"):pid_value>',
         default_media_type='application/json',

diff --git a/rero_ils/es_templates/__init__.py b/rero_ils/es_templates/__init__.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # RERO ILS
-# Copyright (C) 2019 RERO
+# Copyright (C) 2021 RERO
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -19,7 +19,7 @@
 
 
 def list_es_templates():
-    """Elasticsearch Templates path."""
+    """Elasticsearch templates path."""
     return [
         'rero_ils.es_templates'
     ]
diff --git a/rero_ils/modules/cli.py b/rero_ils/modules/cli.py
@@ -73,7 +73,7 @@
 from .items.cli import create_items, reindex_items
 from .loans.cli import create_loans, load_virtua_transactions
 from .monitoring import Monitoring
-from .operation_logs.cli import migrate_virtua_operation_logs
+from .operation_logs.cli import create_operation_logs, dump_operation_logs
 from .patrons.cli import import_users, users_validate
 from .tasks import process_bulk_queue
 from .utils import bulk_load_metadata, bulk_load_pids, bulk_load_pidstore, \
@@ -107,7 +107,8 @@ def fixtures():
 fixtures.add_command(create_patterns)
 fixtures.add_command(create_ill_requests)
 fixtures.add_command(create_collections)
-fixtures.add_command(migrate_virtua_operation_logs)
+fixtures.add_command(create_operation_logs)
+fixtures.add_command(dump_operation_logs)
 
 
 @users.command('confirm')

diff --git a/rero_ils/modules/fetchers.py b/rero_ils/modules/fetchers.py
@@ -27,7 +27,7 @@
 
 
 def id_fetcher(record_uuid, data, provider, pid_key='pid'):
-    """Fetch a Organisation record's identifiers.
+    """Fetch a record's identifier.
 
     :param record_uuid: The record UUID.
     :param data: The record metadata.

diff --git a/rero_ils/modules/monitoring.py b/rero_ils/modules/monitoring.py
@@ -446,10 +446,12 @@ def info(cls, with_deleted=False, difference_db_es=False):
         ).items():
             info[doc_type] = {}
             count_db = cls.get_db_count(doc_type, with_deleted=with_deleted)
+            count_db = count_db if isinstance(count_db, int) else 0
             info[doc_type]['db'] = count_db
             index = endpoint.get('search_index', '')
             if index:
                 count_es = cls.get_es_count(index)
+                count_es = count_es if isinstance(count_es, int) else 0
                 db_es = count_db - count_es
                 info[doc_type]['index'] = index
                 info[doc_type]['es'] = count_es