From 16f28ce66f0294849e16d8ad4d7d46689c643c91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johnny=20Marie=CC=81thoz?= Date: Wed, 19 May 2021 12:27:56 +0200 Subject: [PATCH] operation logs: use an Elasticsearch only resource MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Renames virtua command CLI name. * Fixes monitor view to compute Elasticsearch and DB count diff when the index does not exists. * Creates an operation logs Elasticsearch record class. It creates one index per year. * Denies to all to read one record. * Adds a CLI to dumps the operation logs in a JSON file for backup. * Closes: #1725. Co-Authored-By: Johnny MarieĢthoz --- data/operation_logs.json | 11 +- poetry.lock | 61 +++--- pyproject.toml | 2 +- rero_ils/config.py | 6 +- rero_ils/es_templates/__init__.py | 4 +- rero_ils/modules/cli.py | 5 +- rero_ils/modules/fetchers.py | 2 +- rero_ils/modules/monitoring.py | 2 + rero_ils/modules/operation_logs/api.py | 174 ++++++++++++------ rero_ils/modules/operation_logs/cli.py | 73 ++++---- .../{mappings/v7 => es_templates}/__init__.py | 9 +- .../{mappings => es_templates/v7}/__init__.py | 4 +- .../v7/operation_logs.json} | 6 + rero_ils/modules/operation_logs/extensions.py | 90 +++++++++ .../modules/operation_logs/jsonresolver.py | 28 --- rero_ils/modules/operation_logs/listener.py | 24 +-- rero_ils/modules/operation_logs/models.py | 49 ----- .../modules/operation_logs/permissions.py | 2 +- scripts/setup | 2 +- setup.py | 13 +- .../test_operation_logs_permissions.py | 129 ------------- .../test_operation_logs_rest.py | 94 +++++++--- tests/fixtures/metadata.py | 9 +- .../operation_logs/test_operation_logs_api.py | 78 +++++--- .../test_operation_logs_jsonresolver.py | 47 ----- .../test_operation_logs_mapping.py | 32 +--- tests/ui/test_monitoring.py | 10 +- tests/unit/test_operation_logs_jsonschema.py | 8 +- 28 files changed, 483 insertions(+), 491 deletions(-) rename rero_ils/modules/operation_logs/{mappings/v7 => es_templates}/__init__.py (77%) rename rero_ils/modules/operation_logs/{mappings => es_templates/v7}/__init__.py (88%) rename rero_ils/modules/operation_logs/{mappings/v7/operation_logs/operation_log-v0.0.1.json => es_templates/v7/operation_logs.json} (92%) create mode 100644 rero_ils/modules/operation_logs/extensions.py delete mode 100644 rero_ils/modules/operation_logs/jsonresolver.py delete mode 100644 rero_ils/modules/operation_logs/models.py delete mode 100644 tests/api/operation_logs/test_operation_logs_permissions.py delete mode 100644 tests/ui/operation_logs/test_operation_logs_jsonresolver.py diff --git a/data/operation_logs.json b/data/operation_logs.json index fe51488c70..be8802efb7 100644 --- a/data/operation_logs.json +++ b/data/operation_logs.json @@ -1 +1,10 @@ -[] +[ + { + "record": { + "$ref": "https://ils.rero.ch/api/documents/1" + }, + "operation": "update", + "user_name": "system", + "date": "2021-01-21T09:51:52.879533+00:00" + } +] diff --git a/poetry.lock b/poetry.lock index ae95de9984..862d1877ef 100644 --- a/poetry.lock +++ b/poetry.lock @@ -901,6 +901,7 @@ WTForms = "*" reference = "647dd97880e2105948071b041286f318bf2628f7" type = "git" url = "https://github.com/rero/flask-wiki.git" + [[package]] category = "main" description = "Simple integration of Flask and WTForms." @@ -1329,6 +1330,7 @@ tests = ["mock (>=2.0.0)", "pytest-invenio (>=1.4.0,<1.5.0)", "pytest-mock (>=1. reference = "40e8d2fc3800319179ec84b26bb25ef8711e6356" type = "git" url = "https://github.com/inveniosoftware/invenio-circulation.git" + [[package]] category = "main" description = "Invenio configuration loader." @@ -1531,6 +1533,7 @@ tests = ["check-manifest (>=0.35)", "coverage (>=4.3.4)", "isort (4.2.2)", "mock reference = "fe55e095a8e78b36cfad875f752f2facc2908bae" type = "git" url = "https://github.com/inveniosoftware/invenio-oaiharvester.git" + [[package]] category = "main" description = "Invenio module that implements OAI-PMH server." @@ -1655,12 +1658,12 @@ description = "Invenio-Records is a metadata storage module." name = "invenio-records" optional = false python-versions = "*" -version = "1.4.0" +version = "1.5.0a7" [package.dependencies] arrow = ">=0.16.0" invenio-base = ">=1.2.3" -invenio-celery = ">=1.2.1" +invenio-celery = ">=1.2.2" invenio-i18n = ">=1.2.0" jsonpatch = ">=1.26" jsonref = ">=0.2" @@ -1669,12 +1672,12 @@ jsonschema = ">=3.0.0" [package.extras] admin = ["invenio-admin (>=1.2.1)"] -all = ["Sphinx (>=2.4)", "invenio-admin (>=1.2.1)", "mock (>=3.0.5)", "pytest-invenio (>=1.4.0)"] +all = ["Sphinx (>=2.4)", "invenio-admin (>=1.2.1)", "pytest-invenio (>=1.4.1)"] docs = ["Sphinx (>=2.4)"] -mysql = ["invenio-db (>=1.0.5)"] -postgresql = ["invenio-db (>=1.0.5)"] -sqlite = ["invenio-db (>=1.0.5)"] -tests = ["mock (>=3.0.5)", "pytest-invenio (>=1.4.0)"] +mysql = ["invenio-db (>=1.0.9,<1.1.0)"] +postgresql = ["invenio-db (>=1.0.9,<1.1.0)"] +sqlite = ["invenio-db (>=1.0.9,<1.1.0)"] +tests = ["pytest-invenio (>=1.4.1)"] [[package]] category = "main" @@ -1812,6 +1815,7 @@ tests = ["mock (>=2.0.0)", "pytest-mock (>=1.6.0)", "check-manifest (>=0.35)", " reference = "9baddf0c029e5aa1b75eb7bca0096f56f5f620c6" type = "git" url = "https://github.com/inveniosoftware-contrib/invenio-sip2.git" + [[package]] category = "main" description = "Invenio standard theme." @@ -1864,6 +1868,7 @@ tests = ["pytest-invenio (>=1.4.0)"] reference = "17bf3f045aef278bcf28aaf9a54b9faf3b03e715" type = "git" url = "https://github.com/rero/invenio-userprofiles.git" + [[package]] category = "main" description = "IPython: Productive Interactive Computing" @@ -3136,20 +3141,6 @@ optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" version = "0.10.2" -[[package]] -category = "main" -description = "Traitlets Python configuration system" -name = "traitlets" -optional = false -python-versions = ">=3.7" -version = "5.0.5" - -[package.dependencies] -ipython-genutils = "*" - -[package.extras] -test = ["pytest"] - [[package]] category = "main" description = "Traitlets Python config system" @@ -3169,7 +3160,7 @@ test = ["pytest", "mock"] [[package]] category = "main" description = "Backported and Experimental Type Hints for Python 3.5+" -marker = "python_version < \"3.8\" or python_version >= \"3.7\" and python_version < \"3.8\"" +marker = "python_version < \"3.8\" or python_version >= \"3.7\" and python_version < \"3.8\" or python_version >= \"3.7\" and python_version < \"3.8\" and (python_version < \"3.8\" or python_version >= \"3.7\" and python_version < \"3.8\")" name = "typing-extensions" optional = false python-versions = "*" @@ -3383,7 +3374,7 @@ version = "0.12.0" [[package]] category = "main" description = "Backport of pathlib-compatible object wrapper for zip files" -marker = "python_version < \"3.8\" or python_version >= \"3.7\" and python_version < \"3.8\"" +marker = "python_version < \"3.8\" or python_version >= \"3.7\" and python_version < \"3.8\" or python_version >= \"3.7\" and python_version < \"3.8\" and (python_version < \"3.8\" or python_version >= \"3.7\" and python_version < \"3.8\")" name = "zipp" optional = false python-versions = ">=3.6" @@ -3397,7 +3388,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pyt sip2 = ["invenio-sip2"] [metadata] -content-hash = "e60aadc263ba69db24a7b9d88e221a1d4f1a0092d9d8786dccd3707e48afae2e" +content-hash = "ced160c6ef99817562161d105f61b348dd4ec0d51822a0767e3ec86889a1797a" lock-version = "1.0" python-versions = ">= 3.6, <3.8" @@ -3542,6 +3533,7 @@ click-repl = [ ] colorama = [ {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, + {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, ] coverage = [ {file = "coverage-5.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:b6d534e4b2ab35c9f93f46229363e17f63c53ad01330df9f2d6bd1187e5eaacf"}, @@ -3867,8 +3859,8 @@ invenio-pidstore = [ {file = "invenio_pidstore-1.2.2-py2.py3-none-any.whl", hash = "sha256:960fd76702ebe159392e254ae9222503452383fa1ef0b94673ed0305552917f2"}, ] invenio-records = [ - {file = "invenio-records-1.4.0.tar.gz", hash = "sha256:5387a398dae4271b2fe25008c30f4260aeb80c0f5cd57aa8a9fe6217d7df931d"}, - {file = "invenio_records-1.4.0-py2.py3-none-any.whl", hash = "sha256:d068b54763cc071fec885d842365dd2c8e555a5b6f89cc0e12025d614e582279"}, + {file = "invenio-records-1.5.0a7.tar.gz", hash = "sha256:5eca2262de7eecbf2f68c39694d8786669d129de3699568fef9af383f0bd8425"}, + {file = "invenio_records-1.5.0a7-py2.py3-none-any.whl", hash = "sha256:d9b990ada0afd1c3bda2f1fccfa12c6099aee1f024ddc9e862137f752ba15fbb"}, ] invenio-records-rest = [ {file = "invenio-records-rest-1.8.0.tar.gz", hash = "sha256:70ba741f19f8c9a1ae14a700d82c632175e881fd786ffdc4692f2718482e8dd1"}, @@ -3972,6 +3964,8 @@ lxml = [ {file = "lxml-4.6.3-cp27-cp27m-win_amd64.whl", hash = "sha256:8157dadbb09a34a6bd95a50690595e1fa0af1a99445e2744110e3dca7831c4ee"}, {file = "lxml-4.6.3-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:7728e05c35412ba36d3e9795ae8995e3c86958179c9770e65558ec3fdfd3724f"}, {file = "lxml-4.6.3-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:4bff24dfeea62f2e56f5bab929b4428ae6caba2d1eea0c2d6eb618e30a71e6d4"}, + {file = "lxml-4.6.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:74f7d8d439b18fa4c385f3f5dfd11144bb87c1da034a466c5b5577d23a1d9b51"}, + {file = "lxml-4.6.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:f90ba11136bfdd25cae3951af8da2e95121c9b9b93727b1b896e3fa105b2f586"}, {file = "lxml-4.6.3-cp35-cp35m-win32.whl", hash = "sha256:f2380a6376dfa090227b663f9678150ef27543483055cc327555fb592c5967e2"}, {file = "lxml-4.6.3-cp35-cp35m-win_amd64.whl", hash = "sha256:c4f05c5a7c49d2fb70223d0d5bcfbe474cf928310ac9fa6a7c6dddc831d0b1d4"}, {file = "lxml-4.6.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:d2e35d7bf1c1ac8c538f88d26b396e73dd81440d59c1ef8522e1ea77b345ede4"}, @@ -3998,8 +3992,10 @@ lxml = [ {file = "lxml-4.6.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:884ab9b29feaca361f7f88d811b1eea9bfca36cf3da27768d28ad45c3ee6f969"}, {file = "lxml-4.6.3-cp39-cp39-win32.whl", hash = "sha256:33bb934a044cf32157c12bfcfbb6649807da20aa92c062ef51903415c704704f"}, {file = "lxml-4.6.3-cp39-cp39-win_amd64.whl", hash = "sha256:542d454665a3e277f76954418124d67516c5f88e51a900365ed54a9806122b83"}, + {file = "lxml-4.6.3.tar.gz", hash = "sha256:39b78571b3b30645ac77b95f7c69d1bffc4cf8c3b157c435a34da72e78c82468"}, ] mako = [ + {file = "Mako-1.1.4-py2.py3-none-any.whl", hash = "sha256:aea166356da44b9b830c8023cd9b557fa856bd8b4035d6de771ca027dfc5cc6e"}, {file = "Mako-1.1.4.tar.gz", hash = "sha256:17831f0b7087c313c0ffae2bcbbd3c1d5ba9eeac9c38f2eb7b50e8c99fe9d5ab"}, ] markdown = [ @@ -4178,8 +4174,11 @@ psycopg2-binary = [ {file = "psycopg2_binary-2.8.6-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:2dac98e85565d5688e8ab7bdea5446674a83a3945a8f416ad0110018d1501b94"}, {file = "psycopg2_binary-2.8.6-cp38-cp38-win32.whl", hash = "sha256:bd1be66dde2b82f80afb9459fc618216753f67109b859a361cf7def5c7968729"}, {file = "psycopg2_binary-2.8.6-cp38-cp38-win_amd64.whl", hash = "sha256:8cd0fb36c7412996859cb4606a35969dd01f4ea34d9812a141cd920c3b18be77"}, + {file = "psycopg2_binary-2.8.6-cp39-cp39-macosx_10_9_x86_64.macosx_10_9_intel.macosx_10_10_intel.macosx_10_10_x86_64.whl", hash = "sha256:89705f45ce07b2dfa806ee84439ec67c5d9a0ef20154e0e475e2b2ed392a5b83"}, {file = "psycopg2_binary-2.8.6-cp39-cp39-manylinux1_i686.whl", hash = "sha256:42ec1035841b389e8cc3692277a0bd81cdfe0b65d575a2c8862cec7a80e62e52"}, {file = "psycopg2_binary-2.8.6-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:7312e931b90fe14f925729cde58022f5d034241918a5c4f9797cac62f6b3a9dd"}, + {file = "psycopg2_binary-2.8.6-cp39-cp39-win32.whl", hash = "sha256:6422f2ff0919fd720195f64ffd8f924c1395d30f9a495f31e2392c2efafb5056"}, + {file = "psycopg2_binary-2.8.6-cp39-cp39-win_amd64.whl", hash = "sha256:15978a1fbd225583dd8cdaf37e67ccc278b5abecb4caf6b2d6b8e2b948e953f6"}, ] ptyprocess = [ {file = "ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35"}, @@ -4283,18 +4282,26 @@ pyyaml = [ {file = "PyYAML-5.4.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185"}, {file = "PyYAML-5.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253"}, {file = "PyYAML-5.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541"}, {file = "PyYAML-5.4.1-cp36-cp36m-win32.whl", hash = "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5"}, {file = "PyYAML-5.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df"}, {file = "PyYAML-5.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018"}, {file = "PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"}, {file = "PyYAML-5.4.1-cp37-cp37m-win32.whl", hash = "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b"}, {file = "PyYAML-5.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf"}, {file = "PyYAML-5.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46"}, {file = "PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc"}, {file = "PyYAML-5.4.1-cp38-cp38-win32.whl", hash = "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc"}, {file = "PyYAML-5.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696"}, {file = "PyYAML-5.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77"}, {file = "PyYAML-5.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6"}, {file = "PyYAML-5.4.1-cp39-cp39-win32.whl", hash = "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10"}, {file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"}, {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, @@ -4516,8 +4523,6 @@ toml = [ {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] traitlets = [ - {file = "traitlets-5.0.5-py3-none-any.whl", hash = "sha256:69ff3f9d5351f31a7ad80443c2674b7099df13cc41fc5fa6e2f6d3b0330b0426"}, - {file = "traitlets-5.0.5.tar.gz", hash = "sha256:178f4ce988f69189f7e523337a3e11d91c786ded9360174a3d9ca83e79bc5396"}, {file = "traitlets-4.3.3-py2.py3-none-any.whl", hash = "sha256:70b4c6a1d9019d7b4f6846832288f86998aa3b9207c6821f3578a6a6a467fe44"}, {file = "traitlets-4.3.3.tar.gz", hash = "sha256:d023ee369ddd2763310e4c3eae1ff649689440d4ae59d7485eb4cfbbe3e359f7"}, ] diff --git a/pyproject.toml b/pyproject.toml index e4f9acb6b0..1e31997e73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ invenio-oaiserver = ">=1.2.0,<1.3.0" invenio-pidstore = ">=1.2.1,<1.3.0" invenio-records-rest = ">=1.8.0,<1.9.0" invenio-records-ui= ">=1.2.0,<1.3.0" -invenio-records = ">=1.4.0,<1.6.0" +invenio-records = {version = ">=1.4.0,<1.6.0", allow-prereleases = true} ## Default from Invenio invenio = {version = ">=3.4.0,<5.4.0", extras = ["base", "postgresql", "auth", "elasticsearch7", "docs", "tests" ]} diff --git a/rero_ils/config.py b/rero_ils/config.py index 7b9fdda069..948af6858e 100644 --- a/rero_ils/config.py +++ b/rero_ils/config.py @@ -1613,13 +1613,12 @@ def _(x): action='delete', record=record, cls=TemplatePermission) ), oplg=dict( + # TODO: useless, but required pid_type='oplg', - pid_minter='operation_log_id', + pid_minter='recid', pid_fetcher='operation_log_id', - search_class='rero_ils.modules.operation_logs.api:OperationLogsSearch', search_index='operation_logs', search_type=None, - indexer_class='rero_ils.modules.operation_logs.api:OperationLogsIndexer', record_serializers={ 'application/json': ( 'rero_ils.modules.serializers:json_v1_response' @@ -1638,6 +1637,7 @@ def _(x): }, record_class='rero_ils.modules.operation_logs.api:OperationLog', list_route='/operation_logs/', + # TODO: create a converter for es id, not used for the moment. item_route='/operation_logs/', default_media_type='application/json', diff --git a/rero_ils/es_templates/__init__.py b/rero_ils/es_templates/__init__.py index 69733eafd2..c20fe3984a 100644 --- a/rero_ils/es_templates/__init__.py +++ b/rero_ils/es_templates/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # RERO ILS -# Copyright (C) 2019 RERO +# Copyright (C) 2021 RERO # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Affero General Public License as published by @@ -19,7 +19,7 @@ def list_es_templates(): - """Elasticsearch Templates path.""" + """Elasticsearch templates path.""" return [ 'rero_ils.es_templates' ] diff --git a/rero_ils/modules/cli.py b/rero_ils/modules/cli.py index da8ea0fd3c..5cc5db24b6 100644 --- a/rero_ils/modules/cli.py +++ b/rero_ils/modules/cli.py @@ -76,7 +76,7 @@ from .items.cli import create_items, reindex_items from .loans.cli import create_loans, load_virtua_transactions from .monitoring import Monitoring -from .operation_logs.cli import migrate_virtua_operation_logs +from .operation_logs.cli import create_operation_logs, dump_operation_logs from .patrons.cli import import_users, users_validate from .tasks import process_bulk_queue from .utils import bulk_load_metadata, bulk_load_pids, bulk_load_pidstore, \ @@ -110,7 +110,8 @@ def fixtures(): fixtures.add_command(create_patterns) fixtures.add_command(create_ill_requests) fixtures.add_command(create_collections) -fixtures.add_command(migrate_virtua_operation_logs) +fixtures.add_command(create_operation_logs) +fixtures.add_command(dump_operation_logs) @users.command('confirm') diff --git a/rero_ils/modules/fetchers.py b/rero_ils/modules/fetchers.py index d326ca3178..6a082972e5 100644 --- a/rero_ils/modules/fetchers.py +++ b/rero_ils/modules/fetchers.py @@ -27,7 +27,7 @@ def id_fetcher(record_uuid, data, provider, pid_key='pid'): - """Fetch a Organisation record's identifiers. + """Fetch a record's identifier. :param record_uuid: The record UUID. :param data: The record metadata. diff --git a/rero_ils/modules/monitoring.py b/rero_ils/modules/monitoring.py index 1881782724..0e44a33df2 100644 --- a/rero_ils/modules/monitoring.py +++ b/rero_ils/modules/monitoring.py @@ -446,10 +446,12 @@ def info(cls, with_deleted=False, difference_db_es=False): ).items(): info[doc_type] = {} count_db = cls.get_db_count(doc_type, with_deleted=with_deleted) + count_db = count_db if isinstance(count_db, int) else 0 info[doc_type]['db'] = count_db index = endpoint.get('search_index', '') if index: count_es = cls.get_es_count(index) + count_es = count_es if isinstance(count_es, int) else 0 db_es = count_db - count_es info[doc_type]['index'] = index info[doc_type]['es'] = count_es diff --git a/rero_ils/modules/operation_logs/api.py b/rero_ils/modules/operation_logs/api.py index 9d0ab67259..0d9953f2ec 100644 --- a/rero_ils/modules/operation_logs/api.py +++ b/rero_ils/modules/operation_logs/api.py @@ -17,76 +17,140 @@ """API for manipulating operation_logs.""" -from functools import partial +from elasticsearch.helpers import bulk +from invenio_records.api import RecordBase +from invenio_search import RecordsSearch, current_search_client -from .models import OperationLogIdentifier, OperationLogMetadata, \ - OperationLogOperation -from ..api import IlsRecord, IlsRecordsIndexer, IlsRecordsSearch -from ..fetchers import id_fetcher -from ..minters import id_minter -from ..providers import Provider +from .extensions import DatesExension, IDExtension, ResolveRefsExension +from ..fetchers import FetchedPID -# provider -OperationLogProvider = type( - 'OperationLogProvider', - (Provider,), - dict(identifier=OperationLogIdentifier, pid_type='oplg') -) -# minter -operation_log_id_minter = partial(id_minter, provider=OperationLogProvider) -# fetcher -operation_log_id_fetcher = partial(id_fetcher, provider=OperationLogProvider) +def operation_log_id_fetcher(record_uuid, data): + """Fetch an Organisation record's identifier. -class OperationLogsSearch(IlsRecordsSearch): - """Operation log Search.""" + :param record_uuid: The record UUID. + :param data: The record metadata. + :return: A :data:`rero_ils.modules.fetchers.FetchedPID` instance. + """ + return FetchedPID( + provider=None, + pid_type='oplg', + pid_value=record_uuid + ) - class Meta: - """Search only on operation_log index.""" - index = 'operation_logs' - doc_types = None - fields = ('*', ) - facets = {} - - default_filter = None +class OperationLog(RecordBase): + """OperationLog class.""" + index_name = 'operation_logs' -class OperationLog(IlsRecord): - """OperationLog class.""" + _extensions = [ResolveRefsExension(), DatesExension(), IDExtension()] - minter = operation_log_id_minter - fetcher = operation_log_id_fetcher - provider = OperationLogProvider - model_cls = OperationLogMetadata + @classmethod + def create(cls, data, id_=None, index_refresh='false', **kwargs): + r"""Create a new record instance and store it in elasticsearch. + + :param data: Dict with the record metadata. + :param id_: Specify a UUID to use for the new record, instead of + automatically generated. + :param refresh: If `true` then refresh the affected shards to make + this operation visible to search, if `wait_for` then wait for a + refresh to make this operation visible to search, if `false` + (the default) then do nothing with refreshes. + Valid choices: true, false, wait_for + :returns: A new :class:`Record` instance. + """ + if id_: + data['pid'] = _id + + record = cls( + data, + model=None, + **kwargs + ) + + # Run pre create extensions + for e in cls._extensions: + e.pre_create(record) + + res = current_search_client.index( + index=cls.get_index(record), + body=record.dumps(), + id=record['pid'], + refresh=index_refresh) + + # Run post create extensions + for e in cls._extensions: + e.post_create(record) + return record @classmethod - def get_create_operation_log_by_resource_pid(cls, pid_type, record_pid): - """Return a create operation log for a given resource and pid. + def get_index(cls, data): + """Get the index name given the data. - :param pid_type: resource pid type. - :param record_pid: record pid. - """ - search = OperationLogsSearch() - search = search.filter('term', record__pid=record_pid)\ - .filter('term', record__type=pid_type)\ - .filter('term', operation=OperationLogOperation.CREATE) - oplgs = search.source(['pid']).scan() - try: - return OperationLog.get_record_by_pid(next(oplgs).pid) - except StopIteration: - return None + One index per year is created based on the data date field. + :param data: Dict with the record metadata. + :returns: str, the corresponding index name. + """ + suffix = '-'.join(data.get('date', '').split('-')[0:1]) + return f'{cls.index_name}-{suffix}' -class OperationLogsIndexer(IlsRecordsIndexer): - """Operation log indexing class.""" + @classmethod + def bulk_index(cls, data): + """Bulk indexing. - record_cls = OperationLog + :params data: list of dicts with the record metadata. + """ + actions = [] + for d in data: + d = OperationLog(d) + # Run pre create extensions + for e in cls._extensions: + e.pre_create(d) + + action = { + '_op_type': 'index', + '_index': cls.get_index(d), + '_source': d.dumps(), + '_id': d['pid'] + } + actions.append(action) + n_succeed, errors = bulk(current_search_client, actions) + if n_succeed != len(data): + raise Exception(f'Elasticsearch Indexing Errors: {errors}') - def bulk_index(self, record_id_iterator): - """Bulk index records. + @classmethod + def get_record(cls, _id): + """Retrieve the record by ID. - :param record_id_iterator: Iterator yielding record UUIDs. + Raise a database exception if the record does not exist. + :param id_: record ID. + :returns: The :class:`Record` instance. """ - super(OperationLogsIndexer, self).bulk_index( - record_id_iterator, doc_type='oplg') + # here the elasticsearch get API cannot be used with an index alias + return cls( + next( + RecordsSearch(index=cls.index_name) + .filter('term', _id=_id).scan()) + .to_dict() + ) + + @classmethod + def get_indices(cls): + """Get all index names present in the elasticsearch server.""" + return set([ + v['index'] for v in current_search_client.cat.indices( + index=f'{cls.index_name}*', format='json') + ]) + + @classmethod + def delete_indices(cls): + """Remove all index names present in the elasticsearch server.""" + current_search_client.indices.delete(f'{cls.index_name}*') + return True + + @property + def id(self): + """Get model identifier.""" + return self.get('pid') diff --git a/rero_ils/modules/operation_logs/cli.py b/rero_ils/modules/operation_logs/cli.py index 082a5aba86..bd3e2ceeaf 100644 --- a/rero_ils/modules/operation_logs/cli.py +++ b/rero_ils/modules/operation_logs/cli.py @@ -22,30 +22,26 @@ import json import click -from flask import current_app from flask.cli import with_appcontext +from invenio_search.api import RecordsSearch from rero_ils.modules.operation_logs.api import OperationLog -from rero_ils.modules.operation_logs.models import OperationLogOperation -from rero_ils.modules.utils import extracted_data_from_ref from ..utils import read_json_record -@click.command('migrate_virtua_operation_logs') -@click.option('-v', '--verbose', 'verbose', is_flag=True, default=False) -@click.option('-d', '--debug', 'debug', is_flag=True, default=False) +@click.command('create_operation_logs') @click.option('-l', '--lazy', 'lazy', is_flag=True, default=False) +@click.option('-s', '--batch-size', 'size', type=int, default=10000) @click.argument('infile', type=click.File('r')) @with_appcontext -def migrate_virtua_operation_logs(infile, verbose, debug, lazy): - """Migrate Virtua operation log records in reroils. +def create_operation_logs(infile, lazy, size): + """Load operation log records in reroils. :param infile: Json operation log file. :param lazy: lazy reads file """ - enabled_logs = current_app.config.get('RERO_ILS_ENABLE_OPERATION_LOG') - click.secho('Migrate Virtua operation log records:', fg='green') + click.secho('Load operation log records:', fg='green') if lazy: # try to lazy read json file (slower, better memory management) data = read_json_record(infile) @@ -54,28 +50,41 @@ def migrate_virtua_operation_logs(infile, verbose, debug, lazy): data = json.load(infile) index_count = 0 with click.progressbar(data) as bar: + records = [] for oplg in bar: - try: - operation = oplg.get('operation') - resource = extracted_data_from_ref( - oplg.get('record').get('$ref'), data='resource') - pid_type = enabled_logs.get(resource) - if pid_type and operation == OperationLogOperation.CREATE: - # The virtua create operation log overrides the reroils - # create operation log, the method to use is UPDATE - record_pid = extracted_data_from_ref( - oplg.get('record').get('$ref'), data='pid') + if not (index_count + 1) % size: + OperationLog.bulk_index(records) + records = [] + records.append(oplg) + index_count += 1 + # the rest of the records + if records: + OperationLog.bulk_index(records) + index_count += len(records) + click.echo(f'created {index_count} operation logs.') + + +@click.command('dump_operation_logs') +@click.option('-y', '--year', 'year', type=int) +@click.argument('outfile', type=click.File('w')) +@with_appcontext +def dump_operation_logs(outfile, year): + """Dumps operation log records in a given file. - create_rec = \ - OperationLog.get_create_operation_log_by_resource_pid( - pid_type, record_pid) - if create_rec: - create_rec.update(oplg, dbcommit=True, reindex=True) - elif pid_type and operation == OperationLogOperation.UPDATE: - # The virtua update operation log is a new entry in the - # reroils operation log, the method to use is CREATE - OperationLog.create(data=oplg, dbcommit=True, reindex=True) - except Exception: - pass - index_count += len(data) + :param outfile: JSON operation log output file. + """ + click.secho('Dumps operation log records:', fg='green') + index_name = OperationLog.index_name + if year is not None: + index_name = f'{index_name}-{year}' + search = RecordsSearch(index=index_name) + + index_count = 0 + outfile.write('[\n') + with click.progressbar(search.scan()) as bar: + for oplg in bar: + outfile.write(str(oplg.to_dict())) + outfile.write(',\n') + index_count += 1 + outfile.write('\n]') click.echo(f'created {index_count} operation logs.') diff --git a/rero_ils/modules/operation_logs/mappings/v7/__init__.py b/rero_ils/modules/operation_logs/es_templates/__init__.py similarity index 77% rename from rero_ils/modules/operation_logs/mappings/v7/__init__.py rename to rero_ils/modules/operation_logs/es_templates/__init__.py index e9f3dd93af..28ee0967db 100644 --- a/rero_ils/modules/operation_logs/mappings/v7/__init__.py +++ b/rero_ils/modules/operation_logs/es_templates/__init__.py @@ -15,6 +15,11 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -"""Elasticsearch mappings.""" +"""Elasticsearch templates for Operation log records.""" -from __future__ import absolute_import, print_function + +def list_es_templates(): + """Elasticsearch templates path.""" + return [ + 'rero_ils.modules.operation_logs.es_templates' + ] diff --git a/rero_ils/modules/operation_logs/mappings/__init__.py b/rero_ils/modules/operation_logs/es_templates/v7/__init__.py similarity index 88% rename from rero_ils/modules/operation_logs/mappings/__init__.py rename to rero_ils/modules/operation_logs/es_templates/v7/__init__.py index e9f3dd93af..59643e1c51 100644 --- a/rero_ils/modules/operation_logs/mappings/__init__.py +++ b/rero_ils/modules/operation_logs/es_templates/v7/__init__.py @@ -15,6 +15,4 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -"""Elasticsearch mappings.""" - -from __future__ import absolute_import, print_function +"""ES Templates module for the operation logs.""" diff --git a/rero_ils/modules/operation_logs/mappings/v7/operation_logs/operation_log-v0.0.1.json b/rero_ils/modules/operation_logs/es_templates/v7/operation_logs.json similarity index 92% rename from rero_ils/modules/operation_logs/mappings/v7/operation_logs/operation_log-v0.0.1.json rename to rero_ils/modules/operation_logs/es_templates/v7/operation_logs.json index 2dac7f2459..3bfcd7853f 100644 --- a/rero_ils/modules/operation_logs/mappings/v7/operation_logs/operation_log-v0.0.1.json +++ b/rero_ils/modules/operation_logs/es_templates/v7/operation_logs.json @@ -1,4 +1,7 @@ { + "index_patterns": [ + "operation_logs*" + ], "settings": { "number_of_shards": 8, "number_of_replicas": 1, @@ -60,5 +63,8 @@ "type": "date" } } + }, + "aliases": { + "operation_logs": {} } } diff --git a/rero_ils/modules/operation_logs/extensions.py b/rero_ils/modules/operation_logs/extensions.py new file mode 100644 index 0000000000..9bd835ae1f --- /dev/null +++ b/rero_ils/modules/operation_logs/extensions.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +# +# RERO ILS +# Copyright (C) 2021 RERO +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, version 3 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Operation log record extensions.""" +import uuid +from datetime import datetime + +import pytz +from invenio_records.extensions import RecordExtension + +from ..utils import extracted_data_from_ref + + +class ResolveRefsExension(RecordExtension): + """Replace all $ref values by a dict of pid, type.""" + + mod_type = { + 'documents': 'doc', + 'items': 'item', + 'holdings': 'hold' + } + + def pre_dump(self, record, dumper=None): + """Called before a record is dumped. + + :param record: the record metadata. + :param dumper: the record dumper. + """ + self._resolve_refs(record) + + def _resolve_refs(self, record): + """Recursively replace the $refs. + + Replace in place all $ref to a dict of pid, type values. + + :param record: the record metadata. + """ + for k, v in record.items(): + if isinstance(v, dict): + if v.get('$ref'): + _type = self.mod_type.get( + extracted_data_from_ref(v, data='resource')) + if _type: + resolved = dict( + pid=extracted_data_from_ref(v), + type=_type + ) + record[k] = resolved + else: + self._resolve_refs(v) + + +class IDExtension(RecordExtension): + """Generate an unique ID if does not exists.""" + + def pre_create(self, record): + """Called before a record is committed. + + :param record: the record metadata. + """ + if not record.get('pid'): + record['pid'] = str(uuid.uuid1()) + + +class DatesExension(RecordExtension): + """Set the created and updated date if needed.""" + + def pre_create(self, record): + """Called before a record is committed. + + :param record: the record metadata. + """ + iso_now = pytz.utc.localize(datetime.utcnow()).isoformat() + for date_field in ['_created', '_updated']: + if not record.get(date_field): + record[date_field] = iso_now diff --git a/rero_ils/modules/operation_logs/jsonresolver.py b/rero_ils/modules/operation_logs/jsonresolver.py deleted file mode 100644 index c15108396c..0000000000 --- a/rero_ils/modules/operation_logs/jsonresolver.py +++ /dev/null @@ -1,28 +0,0 @@ -# -*- coding: utf-8 -*- -# -# RERO ILS -# Copyright (C) 2021 RERO -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, version 3 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -"""OperationLog resolver.""" - -import jsonresolver - -from ..jsonresolver import resolve_json_refs - - -@jsonresolver.route('/api/operation_logs/', host='ils.rero.ch') -def operation_log_resolver(pid): - """Resolver for operation_log record.""" - return resolve_json_refs('oplg', pid) diff --git a/rero_ils/modules/operation_logs/listener.py b/rero_ils/modules/operation_logs/listener.py index b792408d0f..eb282a88e0 100644 --- a/rero_ils/modules/operation_logs/listener.py +++ b/rero_ils/modules/operation_logs/listener.py @@ -22,7 +22,6 @@ from flask import current_app from .api import OperationLog -from .models import OperationLogOperation from ..patrons.api import current_librarian from ..utils import extracted_data_from_ref, get_ref_for_pid @@ -36,8 +35,7 @@ def operation_log_record_create(sender, record=None, *args, **kwargs): :param record: the record being created. """ - build_operation_log_record( - record=record, operation=OperationLogOperation.CREATE) + build_operation_log_record(record=record, operation='create') def operation_log_record_update(sender, record=None, *args, **kwargs): @@ -49,14 +47,13 @@ def operation_log_record_update(sender, record=None, *args, **kwargs): :param record: the record being updated. """ - build_operation_log_record( - record=record, operation=OperationLogOperation.UPDATE) + build_operation_log_record(record=record, operation='update') def build_operation_log_record(record=None, operation=None): """Build an operation_log record to load. - :param record: the record being created or updated. + :param record: the record being created. """ if record.get('$schema'): resource_name = extracted_data_from_ref( @@ -65,18 +62,21 @@ def build_operation_log_record(record=None, operation=None): 'RERO_ILS_ENABLE_OPERATION_LOG'): oplg = { 'date': datetime.now(timezone.utc).isoformat(), - 'record': {'$ref': get_ref_for_pid( - record.provider.pid_type, record.get('pid'))}, + 'record': { + '$ref': get_ref_for_pid( + record.provider.pid_type, record.get('pid')) + }, 'operation': operation } if current_librarian: oplg['user'] = { - '$ref': get_ref_for_pid('ptrn', current_librarian.pid)} + '$ref': get_ref_for_pid('ptrn', current_librarian.pid) + } oplg['user_name'] = current_librarian.formatted_name oplg['organisation'] = { '$ref': get_ref_for_pid( - 'org', current_librarian.organisation_pid)} + 'org', current_librarian.organisation_pid) + } else: oplg['user_name'] = 'system' - oplg = OperationLog(oplg) - oplg.create(oplg, dbcommit=True, reindex=True) + OperationLog.create(oplg) diff --git a/rero_ils/modules/operation_logs/models.py b/rero_ils/modules/operation_logs/models.py deleted file mode 100644 index b45d584ed3..0000000000 --- a/rero_ils/modules/operation_logs/models.py +++ /dev/null @@ -1,49 +0,0 @@ -# -*- coding: utf-8 -*- -# -# RERO ILS -# Copyright (C) 2021 RERO -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, version 3 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -"""Define relation between records and buckets.""" - -from __future__ import absolute_import - -from invenio_db import db -from invenio_pidstore.models import RecordIdentifier -from invenio_records.models import RecordMetadataBase - - -class OperationLogIdentifier(RecordIdentifier): - """Sequence generator for OperationLog identifiers.""" - - __tablename__ = 'operation_log_id' - __mapper_args__ = {'concrete': True} - - recid = db.Column( - db.BigInteger().with_variant(db.Integer, 'sqlite'), - primary_key=True, autoincrement=True, - ) - - -class OperationLogMetadata(db.Model, RecordMetadataBase): - """Operation log record metadata.""" - - __tablename__ = 'operation_log_metadata' - - -class OperationLogOperation: - """Enum class to list all possible operations.""" - - CREATE = 'create' - UPDATE = 'update' diff --git a/rero_ils/modules/operation_logs/permissions.py b/rero_ils/modules/operation_logs/permissions.py index 2cc6460963..a25a7bc6b6 100644 --- a/rero_ils/modules/operation_logs/permissions.py +++ b/rero_ils/modules/operation_logs/permissions.py @@ -44,7 +44,7 @@ def read(cls, user, record): :return: "True" if action can be done. """ # all users (lib, sys_lib) can read operation_log records. - return bool(current_librarian) + return False @classmethod def create(cls, user, record=None): diff --git a/scripts/setup b/scripts/setup index d89cc8be88..bd6c262bb3 100755 --- a/scripts/setup +++ b/scripts/setup @@ -511,7 +511,7 @@ then fi info_msg "- Load Virtua operation logs: ${DATA_PATH}/operation_logs.json" -eval ${PREFIX} invenio fixtures migrate_virtua_operation_logs ${DATA_PATH}/operation_logs.json ${CREATE_LAZY} ${DONT_STOP} +eval ${PREFIX} invenio fixtures create_operation_logs ${DATA_PATH}/operation_logs.json ${CREATE_LAZY} ${DONT_STOP} # load legacy circulation transactions from Virtua info_msg "Checkout transactions: ${DATA_PATH}/checkouts.json ${CREATE_LAZY} ${DONT_STOP}" diff --git a/setup.py b/setup.py index c8e1e8b505..e377b48ffe 100644 --- a/setup.py +++ b/setup.py @@ -174,7 +174,6 @@ def run(self): 'patrons = rero_ils.modules.patrons.models', 'templates = rero_ils.modules.templates.models', 'vendors = rero_ils.modules.vendors.models', - 'operation_logs = rero_ils.modules.operation_logs.models', 'selfcheck = rero_ils.modules.selfcheck.models', ], 'invenio_pidstore.minters': [ @@ -202,7 +201,6 @@ def run(self): 'patron_type_id = rero_ils.modules.patron_types.api:patron_type_id_minter', 'template_id = rero_ils.modules.templates.api:template_id_minter', 'vendor_id = rero_ils.modules.vendors.api:vendor_id_minter', - 'operation_log_id = rero_ils.modules.operation_logs.api:operation_log_id_minter', ], 'invenio_pidstore.fetchers': [ 'acq_account_id = rero_ils.modules.acq_accounts.api:acq_account_id_fetcher', @@ -258,7 +256,6 @@ def run(self): 'patrons = rero_ils.modules.patrons.jsonschemas', 'templates = rero_ils.modules.templates.jsonschemas', 'vendors = rero_ils.modules.vendors.jsonschemas', - 'operation_logs = rero_ils.modules.operation_logs.jsonschemas', 'users = rero_ils.modules.users.jsonschemas', ], 'invenio_search.mappings': [ @@ -286,11 +283,12 @@ def run(self): 'patron_types = rero_ils.modules.patron_types.mappings', 'patrons = rero_ils.modules.patrons.mappings', 'templates = rero_ils.modules.templates.mappings', - 'vendors = rero_ils.modules.vendors.mappings', - 'operation_logs = rero_ils.modules.operation_logs.mappings', + 'vendors = rero_ils.modules.vendors.mappings' ], 'invenio_search.templates': [ - 'base-record = rero_ils.es_templates:list_es_templates', + 'rero_ils = rero_ils.es_templates:list_es_templates', + 'operation_logs = rero_ils.modules.operation_logs' + '.es_templates:list_es_templates', ], 'invenio_celery.tasks': [ 'apiharvester = rero_ils.modules.apiharvester.tasks', @@ -326,8 +324,7 @@ def run(self): 'patron_types = rero_ils.modules.patron_types.jsonresolver', 'patrons = rero_ils.modules.patrons.jsonresolver', 'templates = rero_ils.modules.templates.jsonresolver', - 'vendors = rero_ils.modules.vendors.jsonresolver', - 'operation_logs = rero_ils.modules.operation_logs.jsonresolver', + 'vendors = rero_ils.modules.vendors.jsonresolver' ] }, classifiers=[ diff --git a/tests/api/operation_logs/test_operation_logs_permissions.py b/tests/api/operation_logs/test_operation_logs_permissions.py deleted file mode 100644 index cc571df3db..0000000000 --- a/tests/api/operation_logs/test_operation_logs_permissions.py +++ /dev/null @@ -1,129 +0,0 @@ -# -*- coding: utf-8 -*- -# -# RERO ILS -# Copyright (C) 2021 RERO -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, version 3 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -from copy import deepcopy - -import mock -from flask import url_for -from invenio_accounts.testutils import login_user_via_session -from utils import get_json - -from rero_ils.modules.operation_logs.api import OperationLog -from rero_ils.modules.operation_logs.permissions import OperationLogPermission -from rero_ils.modules.utils import get_ref_for_pid - - -def test_operation_logs_permissions_api( - client, document, patron_sion, - librarian_martigny): - """Test operation log permissions api.""" - oplg = OperationLog.get_record_by_pid('1') - - operation_log_permissions_url = url_for( - 'api_blueprint.permissions', - route_name='operation_logs' - ) - operation_log_martigny_permission_url = url_for( - 'api_blueprint.permissions', - route_name='operation_logs', - record_pid=oplg.pid - ) - - # Not logged - res = client.get(operation_log_permissions_url) - assert res.status_code == 401 - - # Logged as patron - login_user_via_session(client, patron_sion.user) - res = client.get(operation_log_permissions_url) - assert res.status_code == 403 - - # Logged as - login_user_via_session(client, librarian_martigny.user) - res = client.get(operation_log_martigny_permission_url) - assert res.status_code == 200 - data = get_json(res) - assert not data['create']['can'] - assert not data['delete']['can'] - assert data['list']['can'] - assert data['read']['can'] - assert not data['update']['can'] - - -def test_operation_logs_permissions(patron_martigny, org_martigny, - librarian_martigny, org_sion, - item_lib_martigny, - system_librarian_martigny): - """Test operation log permissions class.""" - - oplg = OperationLog.get_record_by_pid('1') - - # Anonymous user - assert not OperationLogPermission.list(None, {}) - assert not OperationLogPermission.read(None, {}) - assert not OperationLogPermission.create(None, {}) - assert not OperationLogPermission.update(None, {}) - assert not OperationLogPermission.delete(None, {}) - - # As non Librarian - assert not OperationLogPermission.list(None, oplg) - assert not OperationLogPermission.read(None, oplg) - assert not OperationLogPermission.create(None, oplg) - assert not OperationLogPermission.update(None, oplg) - assert not OperationLogPermission.delete(None, oplg) - - oplg_sion = deepcopy(oplg) - oplg_sion['organisation'] = {'$ref': get_ref_for_pid('org', org_sion.pid)} - oplg_sion = OperationLog.create( - oplg, dbcommit=True, reindex=True, delete_pid=True) - - # As Librarian - with mock.patch( - 'rero_ils.modules.operation_logs.permissions.current_librarian', - librarian_martigny - ): - assert OperationLogPermission.list(None, oplg) - assert OperationLogPermission.read(None, oplg) - assert not OperationLogPermission.create(None, oplg) - assert not OperationLogPermission.update(None, oplg) - assert not OperationLogPermission.delete(None, oplg) - - assert OperationLogPermission.read(None, oplg) - assert not OperationLogPermission.create(None, oplg) - assert not OperationLogPermission.update(None, oplg) - assert not OperationLogPermission.delete(None, oplg) - - assert OperationLogPermission.read(None, oplg_sion) - assert not OperationLogPermission.create(None, oplg_sion) - assert not OperationLogPermission.update(None, oplg_sion) - assert not OperationLogPermission.delete(None, oplg_sion) - - # # As System-librarian - # with mock.patch( - # 'rero_ils.modules.operation_logs.permissions.current_librarian', - # system_librarian_martigny - # ): - # assert not OperationLogPermission.list(None, oplg) - # assert not OperationLogPermission.read(None, oplg) - # assert not OperationLogPermission.create(None, oplg) - # assert not OperationLogPermission.update(None, oplg) - # assert not OperationLogPermission.delete(None, oplg) - - # assert not OperationLogPermission.read(None, oplg_sion) - # assert not OperationLogPermission.create(None, oplg_sion) - # assert not OperationLogPermission.update(None, oplg_sion) - # assert not OperationLogPermission.delete(None, oplg_sion) diff --git a/tests/api/operation_logs/test_operation_logs_rest.py b/tests/api/operation_logs/test_operation_logs_rest.py index df3d95e2a9..f397f2d0b7 100644 --- a/tests/api/operation_logs/test_operation_logs_rest.py +++ b/tests/api/operation_logs/test_operation_logs_rest.py @@ -17,29 +17,75 @@ """Tests REST API operation logs.""" -import mock +from flask import url_for from invenio_accounts.testutils import login_user_via_session +from utils import get_json, postdata -from rero_ils.modules.operation_logs.api import OperationLogsSearch -from rero_ils.modules.operation_logs.models import OperationLogOperation - - -def test_operation_log_entries(client, librarian_martigny, document): - """Test operation log entries after record update.""" - with mock.patch( - 'rero_ils.modules.operation_logs.listener.current_librarian', - librarian_martigny - ): - login_user_via_session(client, librarian_martigny.user) - print('_start_here') - document.update( - document, dbcommit=True, reindex=True) - search = OperationLogsSearch() - results = search.filter( - 'term', - operation=OperationLogOperation.UPDATE).filter( - 'term', record__pid=document.pid).filter( - 'term', user_name=librarian_martigny.formatted_name - ).source().count() - - assert results == 1 + +def test_operation_logs_permissions(client, operation_log, patron_sion, + librarian_martigny, json_header): + """Test operation logs permissions.""" + item_url = url_for('invenio_records_rest.oplg_item', pid_value='1') + item_list = url_for('invenio_records_rest.oplg_list') + + res = client.get(item_url) + assert res.status_code == 404 + + res = client.get(item_list) + assert res.status_code == 401 + + res, _ = postdata( + client, + 'invenio_records_rest.oplg_list', + {} + ) + assert res.status_code == 401 + + res = client.put( + url_for('invenio_records_rest.oplg_item', pid_value='1'), + data={}, + headers=json_header + ) + assert res.status_code == 404 + + res = client.delete(item_url) + assert res.status_code == 404 + + +def test_operation_logs_rest(client, loan_pending_martigny, + librarian_martigny, + json_header): + """Test operation logs REST API.""" + login_user_via_session(client, librarian_martigny.user) + item_url = url_for('invenio_records_rest.oplg_item', pid_value='1') + item_list = url_for('invenio_records_rest.oplg_list') + + res = client.get(item_url) + assert res.status_code == 404 + + res = client.get(item_list) + assert res.status_code == 200 + data = get_json(res) + assert data['hits']['total']['value'] > 0 + pid = data['hits']['hits'][0]['metadata']['pid'] + assert pid + assert data['hits']['hits'][0]['id'] == pid + assert data['hits']['hits'][0]['created'] + assert data['hits']['hits'][0]['updated'] + + res, _ = postdata( + client, + 'invenio_records_rest.oplg_list', + {} + ) + assert res.status_code == 403 + + res = client.put( + url_for('invenio_records_rest.oplg_item', pid_value='1'), + data={}, + headers=json_header + ) + assert res.status_code == 404 + + res = client.delete(item_url) + assert res.status_code == 404 diff --git a/tests/fixtures/metadata.py b/tests/fixtures/metadata.py index 3097909933..beff4a9aa1 100644 --- a/tests/fixtures/metadata.py +++ b/tests/fixtures/metadata.py @@ -31,6 +31,7 @@ from rero_ils.modules.holdings.api import Holding, HoldingsSearch from rero_ils.modules.items.api import Item, ItemsSearch from rero_ils.modules.local_fields.api import LocalField, LocalFieldsSearch +from rero_ils.modules.operation_logs.api import OperationLog from rero_ils.modules.templates.api import Template, TemplatesSearch @@ -1059,6 +1060,12 @@ def local_field_sion(app, org_martigny, document, local_field_sion_data): # --- OPERATION LOGS @pytest.fixture(scope="module") -def operation_log_1_data(data): +def operation_log_data(data): """Load operation log record.""" return deepcopy(data.get('oplg1')) + + +@pytest.fixture(scope="module") +def operation_log(operation_log_data, item_lib_sion): + """Load operation log record.""" + return OperationLog.create(operation_log_data) diff --git a/tests/ui/operation_logs/test_operation_logs_api.py b/tests/ui/operation_logs/test_operation_logs_api.py index 68fb034d27..21371f69d9 100644 --- a/tests/ui/operation_logs/test_operation_logs_api.py +++ b/tests/ui/operation_logs/test_operation_logs_api.py @@ -17,35 +17,61 @@ """Operation logs Record tests.""" -from __future__ import absolute_import, print_function +from copy import deepcopy -from utils import flush_index, get_mapping +import pytest +from invenio_search import current_search -from rero_ils.modules.operation_logs.api import OperationLog, \ - OperationLogsSearch -from rero_ils.modules.operation_logs.api import \ - operation_log_id_fetcher as fetcher -from rero_ils.modules.operation_logs.models import OperationLogOperation +from rero_ils.modules.operation_logs.api import OperationLog -def test_operation_logs_es_mapping(db, item_lib_sion, operation_log_1_data): - """Test operation logs elasticsearch mapping.""" - search = OperationLogsSearch() - mapping = get_mapping(search.Meta.index) - assert mapping - oplg = OperationLog.create(operation_log_1_data, dbcommit=True, - reindex=True, delete_pid=True) - flush_index(OperationLogsSearch.Meta.index) - assert mapping == get_mapping(search.Meta.index) +def test_operation_create(client, es_clear, operation_log_data): + """Test operation logs creation.""" + oplg = OperationLog.create(operation_log_data, index_refresh='wait_for') + assert oplg + assert oplg.id + # need to compare with dumps as it has resolve $refs + data = OperationLog.get_record(oplg.id) + del data['_created'] + del data['_updated'] + assert data == OperationLog(operation_log_data).dumps() + tmp = deepcopy(operation_log_data) + tmp['date'] = '2020-01-21T09:51:52.879533+00:00' + oplg2 = OperationLog.create(tmp, index_refresh='wait_for') + assert OperationLog.get_indices() == set(( + 'operation_logs-2020', + 'operation_logs-2021' + )) + assert OperationLog.get_record(oplg.id) + assert OperationLog.get_record(oplg2.id) + # clean up the index + assert OperationLog.delete_indices() - assert oplg == operation_log_1_data - assert oplg.get('pid') == '7' - oplg = OperationLog.get_record_by_pid('7') - assert oplg == operation_log_1_data - - fetched_pid = fetcher(oplg.id, oplg) - assert fetched_pid.pid_value == '7' - assert fetched_pid.pid_type == 'oplg' - - assert oplg.get('operation') == OperationLogOperation.UPDATE +def test_operation_bulk_index(client, es_clear, operation_log_data): + """Test operation logs bulk creation.""" + data = [] + for date in [ + '2020-01-21T09:51:52.879533+00:00', + '2020-02-21T09:51:52.879533+00:00', + '2020-03-21T09:51:52.879533+00:00', + '2020-04-21T09:51:52.879533+00:00', + '2021-01-21T09:51:52.879533+00:00', + '2021-02-21T09:51:52.879533+00:00' + ]: + tmp = deepcopy(operation_log_data) + tmp['date'] = date + data.append(tmp) + OperationLog.bulk_index(data) + # flush the index for the test + current_search.flush_and_refresh(OperationLog.index_name) + assert OperationLog.get_indices() == set(( + 'operation_logs-2020', + 'operation_logs-2021' + )) + with pytest.raises(Exception) as excinfo: + data[0]['operation'] = dict(name='foo') + OperationLog.bulk_index(data) + assert "BulkIndexError" in str(excinfo.value) + # clean up the index + assert OperationLog.delete_indices() diff --git a/tests/ui/operation_logs/test_operation_logs_jsonresolver.py b/tests/ui/operation_logs/test_operation_logs_jsonresolver.py deleted file mode 100644 index 50c720b51b..0000000000 --- a/tests/ui/operation_logs/test_operation_logs_jsonresolver.py +++ /dev/null @@ -1,47 +0,0 @@ -# -*- coding: utf-8 -*- -# -# RERO ILS -# Copyright (C) 2021 RERO -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, version 3 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . - -"""Operation Logs JSONResolver tests.""" - -import pytest -from invenio_records.api import Record -from jsonref import JsonRefError - -from rero_ils.modules.operation_logs.api import OperationLog - - -def test_operation_log_jsonresolver(item_lib_martigny): - """Test operation logs json resolver.""" - oplg = OperationLog.get_record_by_pid('1') - rec = Record.create({ - 'operation_log': {'$ref': 'https://ils.rero.ch/api/operation_logs/1'} - }) - assert rec.replace_refs().get('operation_log') == \ - {'pid': '1', 'type': 'oplg'} - - # deleted record - oplg.delete() - with pytest.raises(JsonRefError): - rec.replace_refs().dumps() - - # non existing record - rec = Record.create({ - 'operation_logs': { - '$ref': 'https://ils.rero.ch/api/operation_logs/n_e'} - }) - with pytest.raises(JsonRefError): - rec.replace_refs().dumps() diff --git a/tests/ui/operation_logs/test_operation_logs_mapping.py b/tests/ui/operation_logs/test_operation_logs_mapping.py index fe6c9900d5..ddd2f8c128 100644 --- a/tests/ui/operation_logs/test_operation_logs_mapping.py +++ b/tests/ui/operation_logs/test_operation_logs_mapping.py @@ -19,35 +19,13 @@ from utils import get_mapping -from rero_ils.modules.operation_logs.api import OperationLog, \ - OperationLogsSearch -from rero_ils.modules.operation_logs.models import OperationLogOperation +from rero_ils.modules.operation_logs.api import OperationLog -def test_operation_log_es_mapping(item_lib_sion, operation_log_1_data): +def test_operation_log_es_mapping(item_lib_sion, operation_log_data): """Test operation log elasticsearch mapping.""" - search = OperationLogsSearch() - mapping = get_mapping(search.Meta.index) + mapping = get_mapping(OperationLog.index_name) assert mapping OperationLog.create( - operation_log_1_data, - dbcommit=True, - reindex=True, - delete_pid=True) - assert mapping == get_mapping(search.Meta.index) - - count = search.query( - 'query_string', query=OperationLogOperation.CREATE - ).count() - assert count == 3 - - count = search.query( - 'query_string', query=OperationLogOperation.UPDATE - ).count() - assert count == 4 - - count = search.query( - 'match', - **{'user_name': 'updated_user'}).\ - count() - assert count == 1 + operation_log_data) + assert mapping == get_mapping(OperationLog.index_name) diff --git a/tests/ui/test_monitoring.py b/tests/ui/test_monitoring.py index c290e66820..a2bd73dfa4 100644 --- a/tests/ui/test_monitoring.py +++ b/tests/ui/test_monitoring.py @@ -48,7 +48,7 @@ def test_monitoring(app, document_sion_items_data, script_info): ' 0 loc 0 locations 0', ' 0 lofi 0 local_fields 0', ' 0 notif 0 notifications 0', - ' 0 oplg 2 operation_logs 2', + ' -2 oplg 0 operation_logs 2', ' 0 org 0 organisations 0', ' 0 ptre 0 patron_transaction_events 0', ' 0 ptrn 0 patrons 0', @@ -72,6 +72,8 @@ def test_monitoring(app, document_sion_items_data, script_info): assert mon.get_es_count('documents') == 0 assert mon.check() == {'doc': {'db_es': 1}} assert mon.missing('doc') == {'DB': [], 'ES': ['doc3'], 'ES duplicate': []} + # not flushed by default + flush_index('operation_logs') assert mon.info() == { 'acac': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'acq_accounts'}, 'acin': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'acq_invoices'}, @@ -91,7 +93,7 @@ def test_monitoring(app, document_sion_items_data, script_info): 'loc': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'locations'}, 'lofi': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'local_fields'}, 'notif': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'notifications'}, - 'oplg': {'db': 2, 'db-es': 0, 'es': 2, 'index': 'operation_logs'}, + 'oplg': {'db': 0, 'db-es': -2, 'es': 2, 'index': 'operation_logs'}, 'org': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'organisations'}, 'ptre': {'db': 0, 'db-es': 0, 'es': 0, 'index': 'patron_transaction_events'}, @@ -121,10 +123,10 @@ def test_monitoring(app, document_sion_items_data, script_info): doc.reindex() flush_index(DocumentsSearch.Meta.index) assert mon.get_es_count('documents') == 1 - assert mon.check() == {} + assert mon.check() == {'oplg': {'db_es': -2}} assert mon.missing('doc') == {'DB': [], 'ES': [], 'ES duplicate': []} doc.delete(dbcommit=True) assert mon.get_db_count('doc') == 0 assert mon.get_es_count('documents') == 1 - assert mon.check() == {'doc': {'db_es': -1}} + assert mon.check() == {'doc': {'db_es': -1}, 'oplg': {'db_es': -2}} assert mon.missing('doc') == {'DB': ['doc3'], 'ES': [], 'ES duplicate': []} diff --git a/tests/unit/test_operation_logs_jsonschema.py b/tests/unit/test_operation_logs_jsonschema.py index c999e07399..9b99fe6e6c 100644 --- a/tests/unit/test_operation_logs_jsonschema.py +++ b/tests/unit/test_operation_logs_jsonschema.py @@ -24,18 +24,18 @@ from jsonschema.exceptions import ValidationError -def test_required(operation_log_schema, operation_log_1_data): +def test_required(operation_log_schema, operation_log_data): """Test required for operation log jsonschemas.""" - validate(operation_log_1_data, operation_log_schema) + validate(operation_log_data, operation_log_schema) with pytest.raises(ValidationError): validate({}, operation_log_schema) def test_operation_log_all_jsonschema_keys_values( - operation_log_schema, operation_log_1_data): + operation_log_schema, operation_log_data): """Test all keys and values for operation log jsonschema.""" - record = operation_log_1_data + record = operation_log_data validate(record, operation_log_schema) validator = [ {'key': 'pid', 'value': 25},