From 4188f556fb81e269df55d7eb3dd3f1614e850478 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johnny=20Marie=CC=81thoz?= Date: Wed, 19 May 2021 12:27:56 +0200 Subject: [PATCH] operation logs: use an Elasticsearch only resource MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Renames virtua command CLI name. * Fixes monitor view to compute Elasticsearch and DB count diff when the index does not exists. * Creates an operation logs Elasticsearch record class. It creates one index per year. * Denies to all to read one record. * Adds a CLI to dumps the operation logs in a JSON file for backup. * Closes: #1725. Co-Authored-By: Johnny MarieĢthoz --- data/operation_logs.json | 11 +- poetry.lock | 69 +++++-- pyproject.toml | 2 +- rero_ils/config.py | 6 +- rero_ils/es_templates/__init__.py | 4 +- rero_ils/modules/cli.py | 5 +- rero_ils/modules/fetchers.py | 2 +- rero_ils/modules/monitoring.py | 2 + rero_ils/modules/operation_logs/api.py | 174 ++++++++++++------ rero_ils/modules/operation_logs/cli.py | 73 ++++---- .../{mappings/v7 => es_templates}/__init__.py | 9 +- .../{mappings => es_templates/v7}/__init__.py | 4 +- .../v7/operation_logs.json} | 3 + rero_ils/modules/operation_logs/extensions.py | 90 +++++++++ .../modules/operation_logs/jsonresolver.py | 28 --- rero_ils/modules/operation_logs/listener.py | 24 +-- rero_ils/modules/operation_logs/models.py | 49 ----- .../modules/operation_logs/permissions.py | 2 +- scripts/setup | 2 +- setup.py | 13 +- .../test_operation_logs_permissions.py | 129 ------------- .../test_operation_logs_rest.py | 93 +++++++--- tests/fixtures/metadata.py | 9 +- .../operation_logs/test_operation_logs_api.py | 78 +++++--- .../test_operation_logs_jsonresolver.py | 47 ----- .../test_operation_logs_mapping.py | 32 +--- tests/ui/test_monitoring.py | 10 +- tests/unit/test_operation_logs_jsonschema.py | 8 +- 28 files changed, 502 insertions(+), 476 deletions(-) rename rero_ils/modules/operation_logs/{mappings/v7 => es_templates}/__init__.py (77%) rename rero_ils/modules/operation_logs/{mappings => es_templates/v7}/__init__.py (88%) rename rero_ils/modules/operation_logs/{mappings/v7/operation_logs/operation_log-v0.0.1.json => es_templates/v7/operation_logs.json} (95%) create mode 100644 rero_ils/modules/operation_logs/extensions.py delete mode 100644 rero_ils/modules/operation_logs/jsonresolver.py delete mode 100644 rero_ils/modules/operation_logs/models.py delete mode 100644 tests/api/operation_logs/test_operation_logs_permissions.py delete mode 100644 tests/ui/operation_logs/test_operation_logs_jsonresolver.py diff --git a/data/operation_logs.json b/data/operation_logs.json index fe51488c70..be8802efb7 100644 --- a/data/operation_logs.json +++ b/data/operation_logs.json @@ -1 +1,10 @@ -[] +[ + { + "record": { + "$ref": "https://ils.rero.ch/api/documents/1" + }, + "operation": "update", + "user_name": "system", + "date": "2021-01-21T09:51:52.879533+00:00" + } +] diff --git a/poetry.lock b/poetry.lock index 93e9efcbf7..526adb54a9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1658,26 +1658,26 @@ description = "Invenio-Records is a metadata storage module." name = "invenio-records" optional = false python-versions = "*" -version = "1.4.0" +version = "1.5.0a8" [package.dependencies] arrow = ">=0.16.0" invenio-base = ">=1.2.3" -invenio-celery = ">=1.2.1" +invenio-celery = ">=1.2.2" invenio-i18n = ">=1.2.0" jsonpatch = ">=1.26" jsonref = ">=0.2" jsonresolver = ">=0.3.1" -jsonschema = ">=3.0.0" +jsonschema = ">=3.0.0,<4.0.0" [package.extras] admin = ["invenio-admin (>=1.2.1)"] -all = ["Sphinx (>=2.4)", "invenio-admin (>=1.2.1)", "mock (>=3.0.5)", "pytest-invenio (>=1.4.0)"] +all = ["Sphinx (>=2.4)", "invenio-admin (>=1.2.1)", "pytest-invenio (>=1.4.1)"] docs = ["Sphinx (>=2.4)"] -mysql = ["invenio-db (>=1.0.5)"] -postgresql = ["invenio-db (>=1.0.5)"] -sqlite = ["invenio-db (>=1.0.5)"] -tests = ["mock (>=3.0.5)", "pytest-invenio (>=1.4.0)"] +mysql = ["invenio-db (>=1.0.9,<1.1.0)"] +postgresql = ["invenio-db (>=1.0.9,<1.1.0)"] +sqlite = ["invenio-db (>=1.0.9,<1.1.0)"] +tests = ["pytest-invenio (>=1.4.1)"] [[package]] category = "main" @@ -1784,7 +1784,7 @@ description = "Invenio module that add SIP2 communication for self-check" name = "invenio-sip2" optional = true python-versions = "*" -version = "0.5.1" +version = "0.6.0" [package.dependencies] Flask-BabelEx = ">=0.9.4" @@ -1798,6 +1798,7 @@ email-validator = ">=1.0.5" invenio-access = ">=1.3.1" invenio-base = ">=1.2.3" jsonpickle = ">=1.2" +psutil = ">=5.8.0" pycountry = ">=19.7.15" six = ">=1.12.0" @@ -1812,10 +1813,9 @@ sqlite = ["invenio-db (>=1.0.4)"] tests = ["mock (>=2.0.0)", "pytest-mock (>=1.6.0)", "check-manifest (>=0.35)", "coverage (>=4.5.3)", "invenio-app (>=1.2.3)", "invenio-db (>=1.0.4)", "autoflake (>=1.3.1)", "isort (>=5.1.0)", "pydocstyle (>=5.0.0)", "pytest (>=4.6.4,<6.0.0)", "pytest-cache (>=1.0)", "pytest-cov (>=2.7.1)", "pytest-pep8 (>=1.0.6)", "pytest-invenio (>=1.2.2,<1.4.0)"] [package.source] -reference = "9baddf0c029e5aa1b75eb7bca0096f56f5f620c6" +reference = "31b281ea866e71f5f72f19238cdb81e02b06d51c" type = "git" url = "https://github.com/inveniosoftware-contrib/invenio-sip2.git" - [[package]] category = "main" description = "Invenio standard theme." @@ -2461,6 +2461,17 @@ version = "3.0.18" [package.dependencies] wcwidth = "*" +[[package]] +category = "main" +description = "Cross-platform lib for process and system monitoring in Python." +name = "psutil" +optional = true +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "5.8.0" + +[package.extras] +test = ["ipaddress", "mock", "unittest2", "enum34", "pywin32", "wmi"] + [[package]] category = "main" description = "psycopg2 - Python-PostgreSQL Database Adapter" @@ -3396,7 +3407,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pyt sip2 = ["invenio-sip2"] [metadata] -content-hash = "9d0ba480c614c8cdadb957a8b7808c0cb99155f40bb41119ae17ff24bfb7f720" +content-hash = "6aee09d15d6528d11f8950946f3bb2275468bfc49408c780619a6eff427c4d90" lock-version = "1.0" python-versions = ">= 3.6, <3.8" @@ -3867,8 +3878,8 @@ invenio-pidstore = [ {file = "invenio_pidstore-1.2.2-py2.py3-none-any.whl", hash = "sha256:960fd76702ebe159392e254ae9222503452383fa1ef0b94673ed0305552917f2"}, ] invenio-records = [ - {file = "invenio-records-1.4.0.tar.gz", hash = "sha256:5387a398dae4271b2fe25008c30f4260aeb80c0f5cd57aa8a9fe6217d7df931d"}, - {file = "invenio_records-1.4.0-py2.py3-none-any.whl", hash = "sha256:d068b54763cc071fec885d842365dd2c8e555a5b6f89cc0e12025d614e582279"}, + {file = "invenio-records-1.5.0a8.tar.gz", hash = "sha256:e4ed53a6747937029acfc51443de2e06844c169f02db6a7be5dbd83663f099b8"}, + {file = "invenio_records-1.5.0a8-py2.py3-none-any.whl", hash = "sha256:4bc5aca22293db46454c2765f0ce58bf2725937bc6b542e90aa19507a449a9d0"}, ] invenio-records-rest = [ {file = "invenio-records-rest-1.8.0.tar.gz", hash = "sha256:70ba741f19f8c9a1ae14a700d82c632175e881fd786ffdc4692f2718482e8dd1"}, @@ -4154,6 +4165,36 @@ prompt-toolkit = [ {file = "prompt_toolkit-3.0.18-py3-none-any.whl", hash = "sha256:bf00f22079f5fadc949f42ae8ff7f05702826a97059ffcc6281036ad40ac6f04"}, {file = "prompt_toolkit-3.0.18.tar.gz", hash = "sha256:e1b4f11b9336a28fa11810bc623c357420f69dfdb6d2dac41ca2c21a55c033bc"}, ] +psutil = [ + {file = "psutil-5.8.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:0066a82f7b1b37d334e68697faba68e5ad5e858279fd6351c8ca6024e8d6ba64"}, + {file = "psutil-5.8.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:0ae6f386d8d297177fd288be6e8d1afc05966878704dad9847719650e44fc49c"}, + {file = "psutil-5.8.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:12d844996d6c2b1d3881cfa6fa201fd635971869a9da945cf6756105af73d2df"}, + {file = "psutil-5.8.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:02b8292609b1f7fcb34173b25e48d0da8667bc85f81d7476584d889c6e0f2131"}, + {file = "psutil-5.8.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:6ffe81843131ee0ffa02c317186ed1e759a145267d54fdef1bc4ea5f5931ab60"}, + {file = "psutil-5.8.0-cp27-none-win32.whl", hash = "sha256:ea313bb02e5e25224e518e4352af4bf5e062755160f77e4b1767dd5ccb65f876"}, + {file = "psutil-5.8.0-cp27-none-win_amd64.whl", hash = "sha256:5da29e394bdedd9144c7331192e20c1f79283fb03b06e6abd3a8ae45ffecee65"}, + {file = "psutil-5.8.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:74fb2557d1430fff18ff0d72613c5ca30c45cdbfcddd6a5773e9fc1fe9364be8"}, + {file = "psutil-5.8.0-cp36-cp36m-manylinux2010_i686.whl", hash = "sha256:74f2d0be88db96ada78756cb3a3e1b107ce8ab79f65aa885f76d7664e56928f6"}, + {file = "psutil-5.8.0-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:99de3e8739258b3c3e8669cb9757c9a861b2a25ad0955f8e53ac662d66de61ac"}, + {file = "psutil-5.8.0-cp36-cp36m-win32.whl", hash = "sha256:36b3b6c9e2a34b7d7fbae330a85bf72c30b1c827a4366a07443fc4b6270449e2"}, + {file = "psutil-5.8.0-cp36-cp36m-win_amd64.whl", hash = "sha256:52de075468cd394ac98c66f9ca33b2f54ae1d9bff1ef6b67a212ee8f639ec06d"}, + {file = "psutil-5.8.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c6a5fd10ce6b6344e616cf01cc5b849fa8103fbb5ba507b6b2dee4c11e84c935"}, + {file = "psutil-5.8.0-cp37-cp37m-manylinux2010_i686.whl", hash = "sha256:61f05864b42fedc0771d6d8e49c35f07efd209ade09a5afe6a5059e7bb7bf83d"}, + {file = "psutil-5.8.0-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:0dd4465a039d343925cdc29023bb6960ccf4e74a65ad53e768403746a9207023"}, + {file = "psutil-5.8.0-cp37-cp37m-win32.whl", hash = "sha256:1bff0d07e76114ec24ee32e7f7f8d0c4b0514b3fae93e3d2aaafd65d22502394"}, + {file = "psutil-5.8.0-cp37-cp37m-win_amd64.whl", hash = "sha256:fcc01e900c1d7bee2a37e5d6e4f9194760a93597c97fee89c4ae51701de03563"}, + {file = "psutil-5.8.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6223d07a1ae93f86451d0198a0c361032c4c93ebd4bf6d25e2fb3edfad9571ef"}, + {file = "psutil-5.8.0-cp38-cp38-manylinux2010_i686.whl", hash = "sha256:d225cd8319aa1d3c85bf195c4e07d17d3cd68636b8fc97e6cf198f782f99af28"}, + {file = "psutil-5.8.0-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:28ff7c95293ae74bf1ca1a79e8805fcde005c18a122ca983abf676ea3466362b"}, + {file = "psutil-5.8.0-cp38-cp38-win32.whl", hash = "sha256:ce8b867423291cb65cfc6d9c4955ee9bfc1e21fe03bb50e177f2b957f1c2469d"}, + {file = "psutil-5.8.0-cp38-cp38-win_amd64.whl", hash = "sha256:90f31c34d25b1b3ed6c40cdd34ff122b1887a825297c017e4cbd6796dd8b672d"}, + {file = "psutil-5.8.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6323d5d845c2785efb20aded4726636546b26d3b577aded22492908f7c1bdda7"}, + {file = "psutil-5.8.0-cp39-cp39-manylinux2010_i686.whl", hash = "sha256:245b5509968ac0bd179287d91210cd3f37add77dad385ef238b275bad35fa1c4"}, + {file = "psutil-5.8.0-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:90d4091c2d30ddd0a03e0b97e6a33a48628469b99585e2ad6bf21f17423b112b"}, + {file = "psutil-5.8.0-cp39-cp39-win32.whl", hash = "sha256:ea372bcc129394485824ae3e3ddabe67dc0b118d262c568b4d2602a7070afdb0"}, + {file = "psutil-5.8.0-cp39-cp39-win_amd64.whl", hash = "sha256:f4634b033faf0d968bb9220dd1c793b897ab7f1189956e1aa9eae752527127d3"}, + {file = "psutil-5.8.0.tar.gz", hash = "sha256:0c9ccb99ab76025f2f0bbecf341d4656e9c1351db8cc8a03ccd62e318ab4b5c6"}, +] psycopg2-binary = [ {file = "psycopg2-binary-2.8.6.tar.gz", hash = "sha256:11b9c0ebce097180129e422379b824ae21c8f2a6596b159c7659e2e5a00e1aa0"}, {file = "psycopg2_binary-2.8.6-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl", hash = "sha256:d14b140a4439d816e3b1229a4a525df917d6ea22a0771a2a78332273fd9528a4"}, diff --git a/pyproject.toml b/pyproject.toml index ffc585195e..4b0660cacd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ invenio-oaiserver = ">=1.2.0,<1.3.0" invenio-pidstore = ">=1.2.1,<1.3.0" invenio-records-rest = ">=1.8.0,<1.9.0" invenio-records-ui= ">=1.2.0,<1.3.0" -invenio-records = ">=1.4.0,<1.6.0" +invenio-records = {version = ">=1.4.0,<1.6.0", allow-prereleases = true} ## Default from Invenio invenio = {version = ">=3.4.0,<5.4.0", extras = ["base", "postgresql", "auth", "elasticsearch7", "docs", "tests" ]} diff --git a/rero_ils/config.py b/rero_ils/config.py index bebb87b17a..da1fa25aa7 100644 --- a/rero_ils/config.py +++ b/rero_ils/config.py @@ -1613,13 +1613,12 @@ def _(x): action='delete', record=record, cls=TemplatePermission) ), oplg=dict( + # TODO: useless, but required pid_type='oplg', - pid_minter='operation_log_id', + pid_minter='recid', pid_fetcher='operation_log_id', - search_class='rero_ils.modules.operation_logs.api:OperationLogsSearch', search_index='operation_logs', search_type=None, - indexer_class='rero_ils.modules.operation_logs.api:OperationLogsIndexer', record_serializers={ 'application/json': ( 'rero_ils.modules.serializers:json_v1_response' @@ -1638,6 +1637,7 @@ def _(x): }, record_class='rero_ils.modules.operation_logs.api:OperationLog', list_route='/operation_logs/', + # TODO: create a converter for es id, not used for the moment. item_route='/operation_logs/', default_media_type='application/json', diff --git a/rero_ils/es_templates/__init__.py b/rero_ils/es_templates/__init__.py index 69733eafd2..c20fe3984a 100644 --- a/rero_ils/es_templates/__init__.py +++ b/rero_ils/es_templates/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2019 RERO +# Copyright (C) 2021 RERO # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -19,7 +19,7 @@ def list_es_templates(): - """Elasticsearch Templates path.""" + """Elasticsearch templates path.""" return [ 'rero_ils.es_templates' ] diff --git a/rero_ils/modules/cli.py b/rero_ils/modules/cli.py index 7be04b5cd1..71f26b36a8 100644 --- a/rero_ils/modules/cli.py +++ b/rero_ils/modules/cli.py @@ -73,7 +73,7 @@ from .items.cli import create_items, reindex_items from .loans.cli import create_loans, load_virtua_transactions from .monitoring import Monitoring -from .operation_logs.cli import migrate_virtua_operation_logs +from .operation_logs.cli import create_operation_logs, dump_operation_logs from .patrons.cli import import_users, users_validate from .tasks import process_bulk_queue from .utils import bulk_load_metadata, bulk_load_pids, bulk_load_pidstore, \ @@ -107,7 +107,8 @@ def fixtures(): fixtures.add_command(create_patterns) fixtures.add_command(create_ill_requests) fixtures.add_command(create_collections) -fixtures.add_command(migrate_virtua_operation_logs) +fixtures.add_command(create_operation_logs) +fixtures.add_command(dump_operation_logs) @users.command('confirm') diff --git a/rero_ils/modules/fetchers.py b/rero_ils/modules/fetchers.py index d326ca3178..6a082972e5 100644 --- a/rero_ils/modules/fetchers.py +++ b/rero_ils/modules/fetchers.py @@ -27,7 +27,7 @@ def id_fetcher(record_uuid, data, provider, pid_key='pid'): - """Fetch a Organisation record's identifiers. + """Fetch a record's identifier. :param record_uuid: The record UUID. :param data: The record metadata. diff --git a/rero_ils/modules/monitoring.py b/rero_ils/modules/monitoring.py index 1881782724..0e44a33df2 100644 --- a/rero_ils/modules/monitoring.py +++ b/rero_ils/modules/monitoring.py @@ -446,10 +446,12 @@ def info(cls, with_deleted=False, difference_db_es=False): ).items(): info[doc_type] = {} count_db = cls.get_db_count(doc_type, with_deleted=with_deleted) + count_db = count_db if isinstance(count_db, int) else 0 info[doc_type]['db'] = count_db index = endpoint.get('search_index', '') if index: count_es = cls.get_es_count(index) + count_es = count_es if isinstance(count_es, int) else 0 db_es = count_db - count_es info[doc_type]['index'] = index info[doc_type]['es'] = count_es diff --git a/rero_ils/modules/operation_logs/api.py b/rero_ils/modules/operation_logs/api.py index 9d0ab67259..0d9953f2ec 100644 --- a/rero_ils/modules/operation_logs/api.py +++ b/rero_ils/modules/operation_logs/api.py @@ -17,76 +17,140 @@ """API for manipulating operation_logs.""" -from functools import partial +from elasticsearch.helpers import bulk +from invenio_records.api import RecordBase +from invenio_search import RecordsSearch, current_search_client -from .models import OperationLogIdentifier, OperationLogMetadata, \ - OperationLogOperation -from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch -from ..fetchers import id_fetcher -from ..minters import id_minter -from ..providers import Provider +from .extensions import DatesExension, IDExtension, ResolveRefsExension +from ..fetchers import FetchedPID -# provider -OperationLogProvider = type( - 'OperationLogProvider', - (Provider,), - dict(identifier=OperationLogIdentifier, pid_type='oplg') -) -# minter -operation_log_id_minter = partial(id_minter, provider=OperationLogProvider) -# fetcher -operation_log_id_fetcher = partial(id_fetcher, provider=OperationLogProvider) +def operation_log_id_fetcher(record_uuid, data): + """Fetch an Organisation record's identifier. -class OperationLogsSearch(IlsRecordsSearch): - """Operation log Search.""" + :param record_uuid: The record UUID. + :param data: The record metadata. + :return: A :data:`rero_ils.modules.fetchers.FetchedPID` instance. + """ + return FetchedPID( + provider=None, + pid_type='oplg', + pid_value=record_uuid + ) - class Meta: - """Search only on operation_log index.""" - index = 'operation_logs' - doc_types = None - fields = ('*', ) - facets = {} - - default_filter = None +class OperationLog(RecordBase): + """OperationLog class.""" + index_name = 'operation_logs' -class OperationLog(IlsRecord): - """OperationLog class.""" + _extensions = [ResolveRefsExension(), DatesExension(), IDExtension()] - minter = operation_log_id_minter - fetcher = operation_log_id_fetcher - provider = OperationLogProvider - model_cls = OperationLogMetadata + @classmethod + def create(cls, data, id_=None, index_refresh='false', **kwargs): + r"""Create a new record instance and store it in elasticsearch. + + :param data: Dict with the record metadata. + :param id_: Specify a UUID to use for the new record, instead of + automatically generated. + :param refresh: If `true` then refresh the affected shards to make + this operation visible to search, if `wait_for` then wait for a + refresh to make this operation visible to search, if `false` + (the default) then do nothing with refreshes. + Valid choices: true, false, wait_for + :returns: A new :class:`Record` instance. + """ + if id_: + data['pid'] = _id + + record = cls( + data, + model=None, + **kwargs + ) + + # Run pre create extensions + for e in cls._extensions: + e.pre_create(record) + + res = current_search_client.index( + index=cls.get_index(record), + body=record.dumps(), + id=record['pid'], + refresh=index_refresh) + + # Run post create extensions + for e in cls._extensions: + e.post_create(record) + return record @classmethod - def get_create_operation_log_by_resource_pid(cls, pid_type, record_pid): - """Return a create operation log for a given resource and pid. + def get_index(cls, data): + """Get the index name given the data. - :param pid_type: resource pid type. - :param record_pid: record pid. - """ - search = OperationLogsSearch() - search = search.filter('term', record__pid=record_pid)\ - .filter('term', record__type=pid_type)\ - .filter('term', operation=OperationLogOperation.CREATE) - oplgs = search.source(['pid']).scan() - try: - return OperationLog.get_record_by_pid(next(oplgs).pid) - except StopIteration: - return None + One index per year is created based on the data date field. + :param data: Dict with the record metadata. + :returns: str, the corresponding index name. + """ + suffix = '-'.join(data.get('date', '').split('-')[0:1]) + return f'{cls.index_name}-{suffix}' -class OperationLogsIndexer(IlsRecordsIndexer): - """Operation log indexing class.""" + @classmethod + def bulk_index(cls, data): + """Bulk indexing. - record_cls = OperationLog + :params data: list of dicts with the record metadata. + """ + actions = [] + for d in data: + d = OperationLog(d) + # Run pre create extensions + for e in cls._extensions: + e.pre_create(d) + + action = { + '_op_type': 'index', + '_index': cls.get_index(d), + '_source': d.dumps(), + '_id': d['pid'] + } + actions.append(action) + n_succeed, errors = bulk(current_search_client, actions) + if n_succeed != len(data): + raise Exception(f'Elasticsearch Indexing Errors: {errors}') - def bulk_index(self, record_id_iterator): - """Bulk index records. + @classmethod + def get_record(cls, _id): + """Retrieve the record by ID. - :param record_id_iterator: Iterator yielding record UUIDs. + Raise a database exception if the record does not exist. + :param id_: record ID. + :returns: The :class:`Record` instance. """ - super(OperationLogsIndexer, self).bulk_index( - record_id_iterator, doc_type='oplg') + # here the elasticsearch get API cannot be used with an index alias + return cls( + next( + RecordsSearch(index=cls.index_name) + .filter('term', _id=_id).scan()) + .to_dict() + ) + + @classmethod + def get_indices(cls): + """Get all index names present in the elasticsearch server.""" + return set([ + v['index'] for v in current_search_client.cat.indices( + index=f'{cls.index_name}*', format='json') + ]) + + @classmethod + def delete_indices(cls): + """Remove all index names present in the elasticsearch server.""" + current_search_client.indices.delete(f'{cls.index_name}*') + return True + + @property + def id(self): + """Get model identifier.""" + return self.get('pid') diff --git a/rero_ils/modules/operation_logs/cli.py b/rero_ils/modules/operation_logs/cli.py index 082a5aba86..bd3e2ceeaf 100644 --- a/rero_ils/modules/operation_logs/cli.py +++ b/rero_ils/modules/operation_logs/cli.py @@ -22,30 +22,26 @@ import json import click -from flask import current_app from flask.cli import with_appcontext +from invenio_search.api import RecordsSearch from rero_ils.modules.operation_logs.api import OperationLog -from rero_ils.modules.operation_logs.models import OperationLogOperation -from rero_ils.modules.utils import extracted_data_from_ref from ..utils import read_json_record -@click.command('migrate_virtua_operation_logs') -@click.option('-v', '--verbose', 'verbose', is_flag=True, default=False) -@click.option('-d', '--debug', 'debug', is_flag=True, default=False) +@click.command('create_operation_logs') @click.option('-l', '--lazy', 'lazy', is_flag=True, default=False) +@click.option('-s', '--batch-size', 'size', type=int, default=10000) @click.argument('infile', type=click.File('r')) @with_appcontext -def migrate_virtua_operation_logs(infile, verbose, debug, lazy): - """Migrate Virtua operation log records in reroils. +def create_operation_logs(infile, lazy, size): + """Load operation log records in reroils. :param infile: Json operation log file. :param lazy: lazy reads file """ - enabled_logs = current_app.config.get('RERO_ILS_ENABLE_OPERATION_LOG') - click.secho('Migrate Virtua operation log records:', fg='green') + click.secho('Load operation log records:', fg='green') if lazy: # try to lazy read json file (slower, better memory management) data = read_json_record(infile) @@ -54,28 +50,41 @@ def migrate_virtua_operation_logs(infile, verbose, debug, lazy): data = json.load(infile) index_count = 0 with click.progressbar(data) as bar: + records = [] for oplg in bar: - try: - operation = oplg.get('operation') - resource = extracted_data_from_ref( - oplg.get('record').get('$ref'), data='resource') - pid_type = enabled_logs.get(resource) - if pid_type and operation == OperationLogOperation.CREATE: - # The virtua create operation log overrides the reroils - # create operation log, the method to use is UPDATE - record_pid = extracted_data_from_ref( - oplg.get('record').get('$ref'), data='pid') + if not (index_count + 1) % size: + OperationLog.bulk_index(records) + records = [] + records.append(oplg) + index_count += 1 + # the rest of the records + if records: + OperationLog.bulk_index(records) + index_count += len(records) + click.echo(f'created {index_count} operation logs.') + + +@click.command('dump_operation_logs') +@click.option('-y', '--year', 'year', type=int) +@click.argument('outfile', type=click.File('w')) +@with_appcontext +def dump_operation_logs(outfile, year): + """Dumps operation log records in a given file. - create_rec = \ - OperationLog.get_create_operation_log_by_resource_pid( - pid_type, record_pid) - if create_rec: - create_rec.update(oplg, dbcommit=True, reindex=True) - elif pid_type and operation == OperationLogOperation.UPDATE: - # The virtua update operation log is a new entry in the - # reroils operation log, the method to use is CREATE - OperationLog.create(data=oplg, dbcommit=True, reindex=True) - except Exception: - pass - index_count += len(data) + :param outfile: JSON operation log output file. + """ + click.secho('Dumps operation log records:', fg='green') + index_name = OperationLog.index_name + if year is not None: + index_name = f'{index_name}-{year}' + search = RecordsSearch(index=index_name) + + index_count = 0 + outfile.write('[\n') + with click.progressbar(search.scan()) as bar: + for oplg in bar: + outfile.write(str(oplg.to_dict())) + outfile.write(',\n') + index_count += 1 + outfile.write('\n]') click.echo(f'created {index_count} operation logs.') diff --git a/rero_ils/modules/operation_logs/mappings/v7/__init__.py b/rero_ils/modules/operation_logs/es_templates/__init__.py similarity index 77% rename from rero_ils/modules/operation_logs/mappings/v7/__init__.py rename to rero_ils/modules/operation_logs/es_templates/__init__.py index e9f3dd93af..28ee0967db 100644 --- a/rero_ils/modules/operation_logs/mappings/v7/__init__.py +++ b/rero_ils/modules/operation_logs/es_templates/__init__.py @@ -15,6 +15,11 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -"""Elasticsearch mappings.""" +"""Elasticsearch templates for Operation log records.""" -from __future__ import absolute_import, print_function + +def list_es_templates(): + """Elasticsearch templates path.""" + return [ + 'rero_ils.modules.operation_logs.es_templates' + ] diff --git a/rero_ils/modules/operation_logs/mappings/__init__.py b/rero_ils/modules/operation_logs/es_templates/v7/__init__.py similarity index 88% rename from rero_ils/modules/operation_logs/mappings/__init__.py rename to rero_ils/modules/operation_logs/es_templates/v7/__init__.py index e9f3dd93af..59643e1c51 100644 --- a/rero_ils/modules/operation_logs/mappings/__init__.py +++ b/rero_ils/modules/operation_logs/es_templates/v7/__init__.py @@ -15,6 +15,4 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -"""Elasticsearch mappings.""" - -from __future__ import absolute_import, print_function +"""ES Templates module for the operation logs.""" diff --git a/rero_ils/modules/operation_logs/mappings/v7/operation_logs/operation_log-v0.0.1.json b/rero_ils/modules/operation_logs/es_templates/v7/operation_logs.json similarity index 95% rename from rero_ils/modules/operation_logs/mappings/v7/operation_logs/operation_log-v0.0.1.json rename to rero_ils/modules/operation_logs/es_templates/v7/operation_logs.json index fa8c7935cd..d63516cc63 100644 --- a/rero_ils/modules/operation_logs/mappings/v7/operation_logs/operation_log-v0.0.1.json +++ b/rero_ils/modules/operation_logs/es_templates/v7/operation_logs.json @@ -55,5 +55,8 @@ "type": "date" } } + }, + "aliases": { + "operation_logs": {} } } diff --git a/rero_ils/modules/operation_logs/extensions.py b/rero_ils/modules/operation_logs/extensions.py new file mode 100644 index 0000000000..9bd835ae1f --- /dev/null +++ b/rero_ils/modules/operation_logs/extensions.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2021 RERO +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Operation log record extensions.""" +import uuid +from datetime import datetime + +import pytz +from invenio_records.extensions import RecordExtension + +from ..utils import extracted_data_from_ref + + +class ResolveRefsExension(RecordExtension): + """Replace all $ref values by a dict of pid, type.""" + + mod_type = { + 'documents': 'doc', + 'items': 'item', + 'holdings': 'hold' + } + + def pre_dump(self, record, dumper=None): + """Called before a record is dumped. + + :param record: the record metadata. + :param dumper: the record dumper. + """ + self._resolve_refs(record) + + def _resolve_refs(self, record): + """Recursively replace the $refs. + + Replace in place all $ref to a dict of pid, type values. + + :param record: the record metadata. + """ + for k, v in record.items(): + if isinstance(v, dict): + if v.get('$ref'): + _type = self.mod_type.get( + extracted_data_from_ref(v, data='resource')) + if _type: + resolved = dict( + pid=extracted_data_from_ref(v), + type=_type + ) + record[k] = resolved + else: + self._resolve_refs(v) + + +class IDExtension(RecordExtension): + """Generate an unique ID if does not exists.""" + + def pre_create(self, record): + """Called before a record is committed. + + :param record: the record metadata. + """ + if not record.get('pid'): + record['pid'] = str(uuid.uuid1()) + + +class DatesExension(RecordExtension): + """Set the created and updated date if needed.""" + + def pre_create(self, record): + """Called before a record is committed. + + :param record: the record metadata. + """ + iso_now = pytz.utc.localize(datetime.utcnow()).isoformat() + for date_field in ['_created', '_updated']: + if not record.get(date_field): + record[date_field] = iso_now diff --git a/rero_ils/modules/operation_logs/jsonresolver.py b/rero_ils/modules/operation_logs/jsonresolver.py deleted file mode 100644 index 765bbbecf9..0000000000 --- a/rero_ils/modules/operation_logs/jsonresolver.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- -# -# RERO ILS -# Copyright (C) 2021 RERO -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, version 3 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -"""OperationLog resolver.""" - -import jsonresolver - -from ..jsonresolver import resolve_json_refs - - -@jsonresolver.route('/api/operation_logs/', host='bib.rero.ch') -def operation_log_resolver(pid): - """Resolver for operation_log record.""" - return resolve_json_refs('oplg', pid) diff --git a/rero_ils/modules/operation_logs/listener.py b/rero_ils/modules/operation_logs/listener.py index b792408d0f..eb282a88e0 100644 --- a/rero_ils/modules/operation_logs/listener.py +++ b/rero_ils/modules/operation_logs/listener.py @@ -22,7 +22,6 @@ from flask import current_app from .api import OperationLog -from .models import OperationLogOperation from ..patrons.api import current_librarian from ..utils import extracted_data_from_ref, get_ref_for_pid @@ -36,8 +35,7 @@ def operation_log_record_create(sender, record=None, *args, **kwargs): :param record: the record being created. """ - build_operation_log_record( - record=record, operation=OperationLogOperation.CREATE) + build_operation_log_record(record=record, operation='create') def operation_log_record_update(sender, record=None, *args, **kwargs): @@ -49,14 +47,13 @@ def operation_log_record_update(sender, record=None, *args, **kwargs): :param record: the record being updated. """ - build_operation_log_record( - record=record, operation=OperationLogOperation.UPDATE) + build_operation_log_record(record=record, operation='update') def build_operation_log_record(record=None, operation=None): """Build an operation_log record to load. - :param record: the record being created or updated. + :param record: the record being created. """ if record.get('$schema'): resource_name = extracted_data_from_ref( @@ -65,18 +62,21 @@ def build_operation_log_record(record=None, operation=None): 'RERO_ILS_ENABLE_OPERATION_LOG'): oplg = { 'date': datetime.now(timezone.utc).isoformat(), - 'record': {'$ref': get_ref_for_pid( - record.provider.pid_type, record.get('pid'))}, + 'record': { + '$ref': get_ref_for_pid( + record.provider.pid_type, record.get('pid')) + }, 'operation': operation } if current_librarian: oplg['user'] = { - '$ref': get_ref_for_pid('ptrn', current_librarian.pid)} + '$ref': get_ref_for_pid('ptrn', current_librarian.pid) + } oplg['user_name'] = current_librarian.formatted_name oplg['organisation'] = { '$ref': get_ref_for_pid( - 'org', current_librarian.organisation_pid)} + 'org', current_librarian.organisation_pid) + } else: oplg['user_name'] = 'system' - oplg = OperationLog(oplg) - oplg.create(oplg, dbcommit=True, reindex=True) + OperationLog.create(oplg) diff --git a/rero_ils/modules/operation_logs/models.py b/rero_ils/modules/operation_logs/models.py deleted file mode 100644 index b45d584ed3..0000000000 --- a/rero_ils/modules/operation_logs/models.py +++ /dev/null @@ -1,49 +0,0 @@ -# -*- coding: utf-8 -*- -# -# RERO ILS -# Copyright (C) 2021 RERO -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, version 3 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -"""Define relation between records and buckets.""" - -from __future__ import absolute_import - -from invenio_db import db -from invenio_pidstore.models import RecordIdentifier -from invenio_records.models import RecordMetadataBase - - -class OperationLogIdentifier(RecordIdentifier): - """Sequence generator for OperationLog identifiers.""" - - __tablename__ = 'operation_log_id' - __mapper_args__ = {'concrete': True} - - recid = db.Column( - db.BigInteger().with_variant(db.Integer, 'sqlite'), - primary_key=True, autoincrement=True, - ) - - -class OperationLogMetadata(db.Model, RecordMetadataBase): - """Operation log record metadata.""" - - __tablename__ = 'operation_log_metadata' - - -class OperationLogOperation: - """Enum class to list all possible operations.""" - - CREATE = 'create' - UPDATE = 'update' diff --git a/rero_ils/modules/operation_logs/permissions.py b/rero_ils/modules/operation_logs/permissions.py index 2cc6460963..a25a7bc6b6 100644 --- a/rero_ils/modules/operation_logs/permissions.py +++ b/rero_ils/modules/operation_logs/permissions.py @@ -44,7 +44,7 @@ def read(cls, user, record): :return: "True" if action can be done. """ # all users (lib, sys_lib) can read operation_log records. - return bool(current_librarian) + return False @classmethod def create(cls, user, record=None): diff --git a/scripts/setup b/scripts/setup index 713a33379f..bd907cb312 100755 --- a/scripts/setup +++ b/scripts/setup @@ -511,7 +511,7 @@ then fi info_msg "- Load Virtua operation logs: ${DATA_PATH}/operation_logs.json" -eval ${PREFIX} invenio fixtures migrate_virtua_operation_logs ${DATA_PATH}/operation_logs.json ${CREATE_LAZY} ${DONT_STOP} +eval ${PREFIX} invenio fixtures create_operation_logs ${DATA_PATH}/operation_logs.json ${CREATE_LAZY} ${DONT_STOP} # load legacy circulation transactions from Virtua info_msg "Checkout transactions: ${DATA_PATH}/checkouts.json ${CREATE_LAZY} ${DONT_STOP}" diff --git a/setup.py b/setup.py index c8e1e8b505..e377b48ffe 100644 --- a/setup.py +++ b/setup.py @@ -174,7 +174,6 @@ def run(self): 'patrons = rero_ils.modules.patrons.models', 'templates = rero_ils.modules.templates.models', 'vendors = rero_ils.modules.vendors.models', - 'operation_logs = rero_ils.modules.operation_logs.models', 'selfcheck = rero_ils.modules.selfcheck.models', ], 'invenio_pidstore.minters': [ @@ -202,7 +201,6 @@ def run(self): 'patron_type_id = rero_ils.modules.patron_types.api:patron_type_id_minter', 'template_id = rero_ils.modules.templates.api:template_id_minter', 'vendor_id = rero_ils.modules.vendors.api:vendor_id_minter', - 'operation_log_id = rero_ils.modules.operation_logs.api:operation_log_id_minter', ], 'invenio_pidstore.fetchers': [ 'acq_account_id = rero_ils.modules.acq_accounts.api:acq_account_id_fetcher', @@ -258,7 +256,6 @@ def run(self): 'patrons = rero_ils.modules.patrons.jsonschemas', 'templates = rero_ils.modules.templates.jsonschemas', 'vendors = rero_ils.modules.vendors.jsonschemas', - 'operation_logs = rero_ils.modules.operation_logs.jsonschemas', 'users = rero_ils.modules.users.jsonschemas', ], 'invenio_search.mappings': [ @@ -286,11 +283,12 @@ def run(self): 'patron_types = rero_ils.modules.patron_types.mappings', 'patrons = rero_ils.modules.patrons.mappings', 'templates = rero_ils.modules.templates.mappings', - 'vendors = rero_ils.modules.vendors.mappings', - 'operation_logs = rero_ils.modules.operation_logs.mappings', + 'vendors = rero_ils.modules.vendors.mappings' ], 'invenio_search.templates': [ - 'base-record = rero_ils.es_templates:list_es_templates', + 'rero_ils = rero_ils.es_templates:list_es_templates', + 'operation_logs = rero_ils.modules.operation_logs' + '.es_templates:list_es_templates', ], 'invenio_celery.tasks': [ 'apiharvester = rero_ils.modules.apiharvester.tasks', @@ -326,8 +324,7 @@ def run(self): 'patron_types = rero_ils.modules.patron_types.jsonresolver', 'patrons = rero_ils.modules.patrons.jsonresolver', 'templates = rero_ils.modules.templates.jsonresolver', - 'vendors = rero_ils.modules.vendors.jsonresolver', - 'operation_logs = rero_ils.modules.operation_logs.jsonresolver', + 'vendors = rero_ils.modules.vendors.jsonresolver' ] }, classifiers=[ diff --git a/tests/api/operation_logs/test_operation_logs_permissions.py b/tests/api/operation_logs/test_operation_logs_permissions.py deleted file mode 100644 index cc571df3db..0000000000 --- a/tests/api/operation_logs/test_operation_logs_permissions.py +++ /dev/null @@ -1,129 +0,0 @@ -# -*- coding: utf-8 -*- -# -# RERO ILS -# Copyright (C) 2021 RERO -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, version 3 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -from copy import deepcopy - -import mock -from flask import url_for -from invenio_accounts.testutils import login_user_via_session -from utils import get_json - -from rero_ils.modules.operation_logs.api import OperationLog -from rero_ils.modules.operation_logs.permissions import OperationLogPermission -from rero_ils.modules.utils import get_ref_for_pid - - -def test_operation_logs_permissions_api( - client, document, patron_sion, - librarian_martigny): - """Test operation log permissions api.""" - oplg = OperationLog.get_record_by_pid('1') - - operation_log_permissions_url = url_for( - 'api_blueprint.permissions', - route_name='operation_logs' - ) - operation_log_martigny_permission_url = url_for( - 'api_blueprint.permissions', - route_name='operation_logs', - record_pid=oplg.pid - ) - - # Not logged - res = client.get(operation_log_permissions_url) - assert res.status_code == 401 - - # Logged as patron - login_user_via_session(client, patron_sion.user) - res = client.get(operation_log_permissions_url) - assert res.status_code == 403 - - # Logged as - login_user_via_session(client, librarian_martigny.user) - res = client.get(operation_log_martigny_permission_url) - assert res.status_code == 200 - data = get_json(res) - assert not data['create']['can'] - assert not data['delete']['can'] - assert data['list']['can'] - assert data['read']['can'] - assert not data['update']['can'] - - -def test_operation_logs_permissions(patron_martigny, org_martigny, - librarian_martigny, org_sion, - item_lib_martigny, - system_librarian_martigny): - """Test operation log permissions class.""" - - oplg = OperationLog.get_record_by_pid('1') - - # Anonymous user - assert not OperationLogPermission.list(None, {}) - assert not OperationLogPermission.read(None, {}) - assert not OperationLogPermission.create(None, {}) - assert not OperationLogPermission.update(None, {}) - assert not OperationLogPermission.delete(None, {}) - - # As non Librarian - assert not OperationLogPermission.list(None, oplg) - assert not OperationLogPermission.read(None, oplg) - assert not OperationLogPermission.create(None, oplg) - assert not OperationLogPermission.update(None, oplg) - assert not OperationLogPermission.delete(None, oplg) - - oplg_sion = deepcopy(oplg) - oplg_sion['organisation'] = {'$ref': get_ref_for_pid('org', org_sion.pid)} - oplg_sion = OperationLog.create( - oplg, dbcommit=True, reindex=True, delete_pid=True) - - # As Librarian - with mock.patch( - 'rero_ils.modules.operation_logs.permissions.current_librarian', - librarian_martigny - ): - assert OperationLogPermission.list(None, oplg) - assert OperationLogPermission.read(None, oplg) - assert not OperationLogPermission.create(None, oplg) - assert not OperationLogPermission.update(None, oplg) - assert not OperationLogPermission.delete(None, oplg) - - assert OperationLogPermission.read(None, oplg) - assert not OperationLogPermission.create(None, oplg) - assert not OperationLogPermission.update(None, oplg) - assert not OperationLogPermission.delete(None, oplg) - - assert OperationLogPermission.read(None, oplg_sion) - assert not OperationLogPermission.create(None, oplg_sion) - assert not OperationLogPermission.update(None, oplg_sion) - assert not OperationLogPermission.delete(None, oplg_sion) - - # # As System-librarian - # with mock.patch( - # 'rero_ils.modules.operation_logs.permissions.current_librarian', - # system_librarian_martigny - # ): - # assert not OperationLogPermission.list(None, oplg) - # assert not OperationLogPermission.read(None, oplg) - # assert not OperationLogPermission.create(None, oplg) - # assert not OperationLogPermission.update(None, oplg) - # assert not OperationLogPermission.delete(None, oplg) - - # assert not OperationLogPermission.read(None, oplg_sion) - # assert not OperationLogPermission.create(None, oplg_sion) - # assert not OperationLogPermission.update(None, oplg_sion) - # assert not OperationLogPermission.delete(None, oplg_sion) diff --git a/tests/api/operation_logs/test_operation_logs_rest.py b/tests/api/operation_logs/test_operation_logs_rest.py index 0de9c820aa..f397f2d0b7 100644 --- a/tests/api/operation_logs/test_operation_logs_rest.py +++ b/tests/api/operation_logs/test_operation_logs_rest.py @@ -17,28 +17,75 @@ """Tests REST API operation logs.""" -import mock +from flask import url_for from invenio_accounts.testutils import login_user_via_session +from utils import get_json, postdata -from rero_ils.modules.operation_logs.api import OperationLogsSearch -from rero_ils.modules.operation_logs.models import OperationLogOperation - - -def test_operation_log_entries(client, librarian_martigny, document): - """Test operation log entries after record update.""" - with mock.patch( - 'rero_ils.modules.operation_logs.listener.current_librarian', - librarian_martigny - ): - login_user_via_session(client, librarian_martigny.user) - document.update( - document, dbcommit=True, reindex=True) - search = OperationLogsSearch() - results = search.filter( - 'term', - operation=OperationLogOperation.UPDATE).filter( - 'term', record__pid=document.pid).filter( - 'term', user_name=librarian_martigny.formatted_name - ).source().count() - - assert results == 1 + +def test_operation_logs_permissions(client, operation_log, patron_sion, + librarian_martigny, json_header): + """Test operation logs permissions.""" + item_url = url_for('invenio_records_rest.oplg_item', pid_value='1') + item_list = url_for('invenio_records_rest.oplg_list') + + res = client.get(item_url) + assert res.status_code == 404 + + res = client.get(item_list) + assert res.status_code == 401 + + res, _ = postdata( + client, + 'invenio_records_rest.oplg_list', + {} + ) + assert res.status_code == 401 + + res = client.put( + url_for('invenio_records_rest.oplg_item', pid_value='1'), + data={}, + headers=json_header + ) + assert res.status_code == 404 + + res = client.delete(item_url) + assert res.status_code == 404 + + +def test_operation_logs_rest(client, loan_pending_martigny, + librarian_martigny, + json_header): + """Test operation logs REST API.""" + login_user_via_session(client, librarian_martigny.user) + item_url = url_for('invenio_records_rest.oplg_item', pid_value='1') + item_list = url_for('invenio_records_rest.oplg_list') + + res = client.get(item_url) + assert res.status_code == 404 + + res = client.get(item_list) + assert res.status_code == 200 + data = get_json(res) + assert data['hits']['total']['value'] > 0 + pid = data['hits']['hits'][0]['metadata']['pid'] + assert pid + assert data['hits']['hits'][0]['id'] == pid + assert data['hits']['hits'][0]['created'] + assert data['hits']['hits'][0]['updated'] + + res, _ = postdata( + client, + 'invenio_records_rest.oplg_list', + {} + ) + assert res.status_code == 403 + + res = client.put( + url_for('invenio_records_rest.oplg_item', pid_value='1'), + data={}, + headers=json_header + ) + assert res.status_code == 404 + + res = client.delete(item_url) + assert res.status_code == 404 diff --git a/tests/fixtures/metadata.py b/tests/fixtures/metadata.py index 3097909933..beff4a9aa1 100644 --- a/tests/fixtures/metadata.py +++ b/tests/fixtures/metadata.py @@ -31,6 +31,7 @@ from rero_ils.modules.holdings.api import Holding, HoldingsSearch from rero_ils.modules.items.api import Item, ItemsSearch from rero_ils.modules.local_fields.api import LocalField, LocalFieldsSearch +from rero_ils.modules.operation_logs.api import OperationLog from rero_ils.modules.templates.api import Template, TemplatesSearch @@ -1059,6 +1060,12 @@ def local_field_sion(app, org_martigny, document, local_field_sion_data): # --- OPERATION LOGS @pytest.fixture(scope="module") -def operation_log_1_data(data): +def operation_log_data(data): """Load operation log record.""" return deepcopy(data.get('oplg1')) + + +@pytest.fixture(scope="module") +def operation_log(operation_log_data, item_lib_sion): + """Load operation log record.""" + return OperationLog.create(operation_log_data) diff --git a/tests/ui/operation_logs/test_operation_logs_api.py b/tests/ui/operation_logs/test_operation_logs_api.py index 68fb034d27..21371f69d9 100644 --- a/tests/ui/operation_logs/test_operation_logs_api.py +++ b/tests/ui/operation_logs/test_operation_logs_api.py @@ -17,35 +17,61 @@ """Operation logs Record tests.""" -from __future__ import absolute_import, print_function +from copy import deepcopy -from utils import flush_index, get_mapping +import pytest +from invenio_search import current_search -from rero_ils.modules.operation_logs.api import OperationLog, \ - OperationLogsSearch -from rero_ils.modules.operation_logs.api import \ - operation_log_id_fetcher as fetcher -from rero_ils.modules.operation_logs.models import OperationLogOperation +from rero_ils.modules.operation_logs.api import OperationLog -def test_operation_logs_es_mapping(db, item_lib_sion, operation_log_1_data): - """Test operation logs elasticsearch mapping.""" - search = OperationLogsSearch() - mapping = get_mapping(search.Meta.index) - assert mapping - oplg = OperationLog.create(operation_log_1_data, dbcommit=True, - reindex=True, delete_pid=True) - flush_index(OperationLogsSearch.Meta.index) - assert mapping == get_mapping(search.Meta.index) +def test_operation_create(client, es_clear, operation_log_data): + """Test operation logs creation.""" + oplg = OperationLog.create(operation_log_data, index_refresh='wait_for') + assert oplg + assert oplg.id + # need to compare with dumps as it has resolve $refs + data = OperationLog.get_record(oplg.id) + del data['_created'] + del data['_updated'] + assert data == OperationLog(operation_log_data).dumps() + tmp = deepcopy(operation_log_data) + tmp['date'] = '2020-01-21T09:51:52.879533+00:00' + oplg2 = OperationLog.create(tmp, index_refresh='wait_for') + assert OperationLog.get_indices() == set(( + 'operation_logs-2020', + 'operation_logs-2021' + )) + assert OperationLog.get_record(oplg.id) + assert OperationLog.get_record(oplg2.id) + # clean up the index + assert OperationLog.delete_indices() - assert oplg == operation_log_1_data - assert oplg.get('pid') == '7' - oplg = OperationLog.get_record_by_pid('7') - assert oplg == operation_log_1_data - - fetched_pid = fetcher(oplg.id, oplg) - assert fetched_pid.pid_value == '7' - assert fetched_pid.pid_type == 'oplg' - - assert oplg.get('operation') == OperationLogOperation.UPDATE +def test_operation_bulk_index(client, es_clear, operation_log_data): + """Test operation logs bulk creation.""" + data = [] + for date in [ + '2020-01-21T09:51:52.879533+00:00', + '2020-02-21T09:51:52.879533+00:00', + '2020-03-21T09:51:52.879533+00:00', + '2020-04-21T09:51:52.879533+00:00', + '2021-01-21T09:51:52.879533+00:00', + '2021-02-21T09:51:52.879533+00:00' + ]: + tmp = deepcopy(operation_log_data) + tmp['date'] = date + data.append(tmp) + OperationLog.bulk_index(data) + # flush the index for the test + current_search.flush_and_refresh(OperationLog.index_name) + assert OperationLog.get_indices() == set(( + 'operation_logs-2020', + 'operation_logs-2021' + )) + with pytest.raises(Exception) as excinfo: + data[0]['operation'] = dict(name='foo') + OperationLog.bulk_index(data) + assert "BulkIndexError" in str(excinfo.value) + # clean up the index + assert OperationLog.delete_indices() diff --git a/tests/ui/operation_logs/test_operation_logs_jsonresolver.py b/tests/ui/operation_logs/test_operation_logs_jsonresolver.py deleted file mode 100644 index 12a20f9e13..0000000000 --- a/tests/ui/operation_logs/test_operation_logs_jsonresolver.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- -# -# RERO ILS -# Copyright (C) 2021 RERO -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, version 3 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -"""Operation Logs JSONResolver tests.""" - -import pytest -from invenio_records.api import Record -from jsonref import JsonRefError - -from rero_ils.modules.operation_logs.api import OperationLog - - -def test_operation_log_jsonresolver(item_lib_martigny): - """Test operation logs json resolver.""" - oplg = OperationLog.get_record_by_pid('1') - rec = Record.create({ - 'operation_log': {'$ref': 'https://bib.rero.ch/api/operation_logs/1'} - }) - assert rec.replace_refs().get('operation_log') == \ - {'pid': '1', 'type': 'oplg'} - - # deleted record - oplg.delete() - with pytest.raises(JsonRefError): - rec.replace_refs().dumps() - - # non existing record - rec = Record.create({ - 'operation_logs': { - '$ref': 'https://bib.rero.ch/api/operation_logs/n_e'} - }) - with pytest.raises(JsonRefError): - rec.replace_refs().dumps() diff --git a/tests/ui/operation_logs/test_operation_logs_mapping.py b/tests/ui/operation_logs/test_operation_logs_mapping.py index fe6c9900d5..ddd2f8c128 100644 --- a/tests/ui/operation_logs/test_operation_logs_mapping.py +++ b/tests/ui/operation_logs/test_operation_logs_mapping.py @@ -19,35 +19,13 @@ from utils import get_mapping -from rero_ils.modules.operation_logs.api import OperationLog, \ - OperationLogsSearch -from rero_ils.modules.operation_logs.models import OperationLogOperation +from rero_ils.modules.operation_logs.api import OperationLog -def test_operation_log_es_mapping(item_lib_sion, operation_log_1_data): +def test_operation_log_es_mapping(item_lib_sion, operation_log_data): """Test operation log elasticsearch mapping.""" - search = OperationLogsSearch() - mapping = get_mapping(search.Meta.index) + mapping = get_mapping(OperationLog.index_name) assert mapping OperationLog.create( - operation_log_1_data, - dbcommit=True, - reindex=True, - delete_pid=True) - assert mapping == get_mapping(search.Meta.index) - - count = search.query( - 'query_string', query=OperationLogOperation.CREATE - ).count() - assert count == 3 - - count = search.query( - 'query_string', query=OperationLogOperation.UPDATE - ).count() - assert count == 4 - - count = search.query( - 'match', - **{'user_name': 'updated_user'}).\ - count() - assert count == 1 + operation_log_data) + assert mapping == get_mapping(OperationLog.index_name) diff --git a/tests/ui/test_monitoring.py b/tests/ui/test_monitoring.py index c290e66820..a2bd73dfa4 100644 --- a/tests/ui/test_monitoring.py +++ b/tests/ui/test_monitoring.py @@ -48,7 +48,7 @@ def test_monitoring(app, document_sion_items_data, script_info): ' 0 loc 0 locations 0', ' 0 lofi 0 local_fields 0', ' 0 notif 0 notifications 0', - ' 0 oplg 2 operation_logs 2', + ' -2 oplg 0 operation_logs 2', ' 0 org 0 organisations 0', ' 0 ptre 0 patron_transaction_events 0', ' 0 ptrn 0 patrons 0', @@ -72,6 +72,8 @@ def test_monitoring(app, document_sion_items_data, script_info): assert mon.get_es_count('documents') == 0 assert mon.check() == {'doc': {'db_es': 1}} assert mon.missing('doc') == {'DB': [], 'ES': ['doc3'], 'ES duplicate': []} + # not flushed by default + flush_index('operation_logs') assert mon.info() == { 'acac': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'acq_accounts'}, 'acin': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'acq_invoices'}, @@ -91,7 +93,7 @@ def test_monitoring(app, document_sion_items_data, script_info): 'loc': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'locations'}, 'lofi': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'local_fields'}, 'notif': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'notifications'}, - 'oplg': {'db': 2, 'db-es': 0, 'es': 2, 'index': 'operation_logs'}, + 'oplg': {'db': 0, 'db-es': -2, 'es': 2, 'index': 'operation_logs'}, 'org': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'organisations'}, 'ptre': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'patron_transaction_events'}, @@ -121,10 +123,10 @@ def test_monitoring(app, document_sion_items_data, script_info): doc.reindex() flush_index(DocumentsSearch.Meta.index) assert mon.get_es_count('documents') == 1 - assert mon.check() == {} + assert mon.check() == {'oplg': {'db_es': -2}} assert mon.missing('doc') == {'DB': [], 'ES': [], 'ES duplicate': []} doc.delete(dbcommit=True) assert mon.get_db_count('doc') == 0 assert mon.get_es_count('documents') == 1 - assert mon.check() == {'doc': {'db_es': -1}} + assert mon.check() == {'doc': {'db_es': -1}, 'oplg': {'db_es': -2}} assert mon.missing('doc') == {'DB': ['doc3'], 'ES': [], 'ES duplicate': []} diff --git a/tests/unit/test_operation_logs_jsonschema.py b/tests/unit/test_operation_logs_jsonschema.py index c999e07399..9b99fe6e6c 100644 --- a/tests/unit/test_operation_logs_jsonschema.py +++ b/tests/unit/test_operation_logs_jsonschema.py @@ -24,18 +24,18 @@ from jsonschema.exceptions import ValidationError -def test_required(operation_log_schema, operation_log_1_data): +def test_required(operation_log_schema, operation_log_data): """Test required for operation log jsonschemas.""" - validate(operation_log_1_data, operation_log_schema) + validate(operation_log_data, operation_log_schema) with pytest.raises(ValidationError): validate({}, operation_log_schema) def test_operation_log_all_jsonschema_keys_values( - operation_log_schema, operation_log_1_data): + operation_log_schema, operation_log_data): """Test all keys and values for operation log jsonschema.""" - record = operation_log_1_data + record = operation_log_data validate(record, operation_log_schema) validator = [ {'key': 'pid', 'value': 25},