modules: create export module

pyproject.toml

@@ -11,6 +11,7 @@ PyYAML = "^6.0"
 invenio-search = {version = "^1.4.2", extras = ["elasticsearch7"]}
 jsonpatch = "^1.32"
 invenio-db = "^1.0.14"
+invenio-records-rest = "*"
 
 [tool.poetry.dev-dependencies]
 safety = "^1.10.3"
@@ -30,3 +31,10 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry.plugins."flask.commands"]
 rero = "rero_invenio_base.cli:rero"
+
+[tool.poetry.plugins."invenio_base.apps"]
+rero-invenio-base-export = "rero_invenio_base.modules.export.ext:ReroInvenioBaseExportApp"
+
+[tool.poetry.plugins."invenio_base.api_blueprints"]
+rero_ils_exports = "rero_invenio_base.modules.export.views:create_blueprint_from_app"
+

rero_invenio_base/modules/__init__.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+#
+# RERO Invenio Base
+# Copyright (C) 2022 RERO.
+# Copyright (C) 2022 UCLouvain.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""RERO Invenio Base Modules extension."""

rero_invenio_base/modules/export/__init__.py

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+#
+# RERO Invenio Base
+# Copyright (C) 2022 RERO.
+# Copyright (C) 2022 UCLouvain.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""RERO Invenio Base export module extension."""
+
+from rero_invenio_base.modules.export.proxies import current_export
+
+__all__ = ('current_export',)

rero_invenio_base/modules/export/config.py

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+#
+# RERO Invenio Base
+# Copyright (C) 2022 RERO.
+# Copyright (C) 2022 UCLouvain.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""RERO Invenio Base export module configuration file."""
+
+"""
+  This module must be used to create dynamic export route for resource
+  configured by the `invenio-records-rest`. Exports endpoints will provide
+  streamed content. The content is based on an ElasticSearch search result ;
+  this result is processed using ElasticSearch `scan()` method tu fully
+  implement streamed result.
+
+  Each configured endpoint add a flask blueprint endpoint accessible using the
+  `/export/{resource_list_route/` url.
+
+
+.. code-block:: python
+
+RERO_INVENIO_BASE_EXPORT_REST_ENDPOINTS = dict(
+    loan=dict(
+        resource={invenio-record-rest_resource_configuration_endpoint},
+        default_media_type='text/csv',
+        search_serializers={
+            'text/csv': 'rero_ils.modules.loans.serializers:csv_stream_search',
+        },
+        search_serializers_aliases={
+            'csv': 'text/csv'
+        }
+    )
+)
+
+:param resource: Pointer to the resource rest configuration endpoint from
+    `invenio-record-rest`. Check `https://github.com/inveniosoftware/invenio-
+    records-rest/blob/master/invenio_records_rest/config.py` to get correct
+    resource configuration.
+
+:param search_serializers: It contains the list of records serializers for all
+    supported format. This configuration differ from the previous because in
+    this case it handle a list of records resulted by a search query instead of
+    a single record.
+
+:param search_serializers_aliases: A mapping of values of the defined query arg
+    (see `config.REST_MIMETYPE_QUERY_ARG_NAME`) to valid mimetypes for records
+    search serializers: dict(alias -> mimetype).
+
+"""

rero_invenio_base/modules/export/ext.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+#
+# RERO Invenio Base
+# Copyright (C) 2022 RERO.
+# Copyright (C) 2022 UCLouvain.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""RERO Invenio base module declaration for streamed exports."""
+
+
+class ReroInvenioBaseExportApp:
+    """RERO Invenio base export app."""
+
+    def __init__(self, app=None):
+        """Extension initialization."""
+        if app:
+            self.app = app
+            self.init_app(app)
+
+    def init_app(self, app):
+        """Flask application initialization."""
+        self.init_config(app)
+        app.extensions['rero_invenio_base_exports'] = self
+
+    def init_config(self, app):
+        """Initialize configuration."""
+        for k in dir(app.config):
+            if k.startswith('RERO_INVENIO_BASE_EXPORT'):
+                app.config.setdefault(k, getattr(app.config, k))

rero_invenio_base/modules/export/proxies.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+#
+# RERO Invenio Base
+# Copyright (C) 2022 RERO.
+# Copyright (C) 2022 UCLouvain.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""Helper proxy."""
+
+from flask import current_app
+from werkzeug.local import LocalProxy
+
+current_export = LocalProxy(
+    lambda: current_app.extensions["rero_invenio_base_exports"]
+)
+"""Helper proxy to get the current 'RERO Invenio base' exports extension."""

rero_invenio_base/modules/export/views.py

@@ -0,0 +1,133 @@
+# -*- coding: utf-8 -*-
+#
+# RERO Invenio Base
+# Copyright (C) 2022 RERO.
+# Copyright (C) 2022 UCLouvain.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, version 3 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+"""RERO Invenio Base exports views."""
+from copy import deepcopy
+from functools import partial
+
+from flask import Blueprint
+from invenio_pidstore import current_pidstore
+from invenio_records_rest.query import es_search_factory
+from invenio_records_rest.utils import obj_or_import_string
+from invenio_records_rest.views import need_record_permission
+from invenio_rest import ContentNegotiatedMethodView
+from invenio_search import RecordsSearch
+
+
+def create_blueprint_from_app(app):
+    """Create RERO exports blueprint from a Flask application.
+
+    .. note::
+        This function assumes that the application has loaded all extensions
+        that want to register REST endpoints via the ``RECORDS_REST_ENDPOINTS``
+        configuration variable.
+
+    :params app: A Flask application.
+    :returns: Configured blueprint.
+    """
+    api_blueprint = Blueprint('api_exports', __name__, url_prefix='')
+    endpoints = app.config.get('RERO_INVENIO_BASE_EXPORT_REST_ENDPOINTS', {})
+    for key, config in endpoints.items():
+        copy_config = deepcopy(config)
+        resource_config = copy_config.pop('resource', {})
+        route_config = {**resource_config, **copy_config}  # merging dict
+        rule = create_export_url_route(key, **route_config)
+        api_blueprint.add_url_rule(**rule)
+    return api_blueprint
+
+
+def create_export_url_route(endpoint, default_media_type=None,
+                            list_permission_factory_imp=None,
+                            list_route=None, pid_fetcher=None,
+                            search_class=None,
+                            search_factory_imp=None,
+                            search_serializers=None,
+                            search_serializers_aliases=None, **kwargs):
+    """Create Werkzeug URL rule for resource streamed export.
+
+    :param kwargs: all argument necessary to build the flask endpoint.
+    :returns: a configuration dict who can be passed as keywords argument to
+        ``Blueprint.add_url_rule``.
+    """
+    assert list_route
+    assert search_serializers
+
+    # BUILD LIST_ROUTE AND VIEW_NAME
+    #   Override the resource `list_route` adding an "export" prefix.
+    #   NOTE: Using REST guidelines it should be best to build the path using
+    #     "export" as url suffix ; but it can't be done here because this url
+    #     is already used for record serialization.
+    view_name = f'{endpoint}_export'
+    list_route = f'/export{list_route}'
+
+    # ACCESS PERMISSIONS
+    #   Permission to access to any export route are the same as list resource
+    #   records permission.
+    permission_factory = obj_or_import_string(list_permission_factory_imp)
+
+    search_class = obj_or_import_string(search_class, default=RecordsSearch)
+
+    export_view = ExportResource.as_view(
+        view_name,
+        default_media_type=default_media_type,
+        permission_factory=permission_factory,
+        pid_fetcher=pid_fetcher,
+        search_class=search_class,
+        search_serializers=search_serializers,
+        serializers_query_aliases=search_serializers_aliases,
+        search_factory=obj_or_import_string(
+            search_factory_imp, default=es_search_factory)
+    )
+    return {'rule': list_route, 'view_func': export_view}
+
+
+class ExportResource(ContentNegotiatedMethodView):
+    """Resource for records streamed exports."""
+
+    def __init__(self, default_media_type=None, permission_factory=None,
+                 pid_fetcher=None, search_class=None, search_factory=None,
+                 search_serializers=None, serializers_query_aliases=None,
+                 **kwargs):
+        """Init magic method."""
+        serializers = {
+            mime: obj_or_import_string(search_obj)
+            for mime, search_obj in search_serializers.items() or {}.items()
+        }
+        super().__init__(
+            method_serializers={'GET': serializers},
+            serializers_query_aliases=serializers_query_aliases,
+            default_method_media_type={'GET': default_media_type},
+            default_media_type=default_media_type,
+            **kwargs
+        )
+        self.permission_factory = permission_factory
+        self.pid_fetcher = current_pidstore.fetchers[pid_fetcher]
+        self.search_class = search_class
+        self.search_factory = partial(search_factory, self)
+
+    @need_record_permission('permission_factory')
+    def get(self, **kwargs):
+        """Implements GET /export/{resource_list_name}."""
+        search_obj = self.search_class()
+        search = search_obj.with_preference_param().params(version=True)
+        search, _ = self.search_factory(search)
+
+        return self.make_response(
+            pid_fetcher=None,
+            search_result=search.scan()
+        )

run-tests.sh

@@ -33,7 +33,19 @@ function cleanup() {
     echo "Done"
 }
 trap cleanup EXIT
-safety check -i 42194
+
+# Check vulnerabilities:
+#
+# Exception are
+# +============================+===========+==========================+==========+
+# | package                    | installed | affected                 | ID       |
+# +============================+===========+==========================+==========+
+# | kombu                      | 5.1.0     | <5.2.1                   | 42497    |
+# | celery                     | 5.0.2     | <5.2.0                   | 42498    |
+# | celery                     | 5.0.2     | <5.2.2                   | 43738    |
+# +==============================================================================+
+safety check -i 42194 -i 42497 -i 42498 -i 43738
+
 flask rero utils check_license check_license_config.yml
 pydocstyle rero_ils tests docs
 isort --check-only --diff rero_invenio_base tests

tests/conftest.py

@@ -23,14 +23,17 @@
 
 
 import contextlib
+import copy
 
 import pytest
 from click.testing import CliRunner
 from flask import Flask
 from invenio_db import InvenioDB
+from invenio_records_rest import config as _config
 from invenio_search import InvenioSearch, current_search_client
 
 from rero_invenio_base import REROInvenioBase
+from rero_invenio_base.modules.export.ext import ReroInvenioBaseExportApp
 
 
 @pytest.fixture(scope='function')
@@ -79,7 +82,12 @@ def create_app(instance_path):
     def factory(**config):
         app = Flask('testapp', instance_path=instance_path)
         app.config.update(**config)
+        app.config.update(
+            RECORDS_REST_ENDPOINTS=copy.deepcopy(
+                _config.RECORDS_REST_ENDPOINTS)
+        )
         REROInvenioBase(app)
+        ReroInvenioBaseExportApp(app)
         InvenioDB(app)
         InvenioSearch(app)
         return app