diff --git a/edi_storage_oca/README.rst b/edi_storage_oca/README.rst new file mode 100644 index 0000000000..2a571d890f --- /dev/null +++ b/edi_storage_oca/README.rst @@ -0,0 +1,114 @@ +=========================== +EDI Storage backend support +=========================== + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:58511c29ba3a21a1216a155c53ffb1da90ba5561d407af23213e4207b9a0bcaf + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png + :target: https://odoo-community.org/page/development-status + :alt: Beta +.. |badge2| image:: https://img.shields.io/badge/licence-LGPL--3-blue.png + :target: http://www.gnu.org/licenses/lgpl-3.0-standalone.html + :alt: License: LGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fedi-lightgray.png?logo=github + :target: https://github.com/OCA/edi/tree/12.0/edi_storage_oca + :alt: OCA/edi +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/edi-12-0/edi-12-0-edi_storage_oca + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/edi&target_branch=12.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +Allow exchange files using storage backends from `OCA/storage`. + +This module adds a storage backend relation on the EDI backend. +There you can configure the backend to be used (most often and SFTP) +and the paths where to read or put files. + +Often the convention when exchanging files via SFTP +is to have one input forder (to receive files) +and an output folder (to send files). + +Inside this folder you have this hierarchy:: + + input/output folder + |- pending + |- done + |- error + +* `pending` folder contains files that have been just sent +* `done` folder contains files that have been processes successfully +* `error` folder contains files with errors and cannot be processed + +The storage handlers take care of reading files and putting files +in/from the right place and update exchange records data accordingly. + +**Table of contents** + +.. contents:: + :local: + +Usage +===== + +Go to "EDI -> EDI backend" then configure your backend to use a storage backend. + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +~~~~~~~ + +* ACSONE + +Contributors +~~~~~~~~~~~~ + +* Simone Orsi +* Foram Shah +* Lois Rilo +* `Trobz `_: + + * Thien + + +Other credits +~~~~~~~~~~~~~ + +The backport of this module from 14.0 to 12.0 was financially supported by Camptocamp + +Maintainers +~~~~~~~~~~~ + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +This module is part of the `OCA/edi `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/edi_storage_oca/__init__.py b/edi_storage_oca/__init__.py new file mode 100644 index 0000000000..f24d3e2426 --- /dev/null +++ b/edi_storage_oca/__init__.py @@ -0,0 +1,2 @@ +from . import components +from . import models diff --git a/edi_storage_oca/__manifest__.py b/edi_storage_oca/__manifest__.py new file mode 100644 index 0000000000..7153865ac4 --- /dev/null +++ b/edi_storage_oca/__manifest__.py @@ -0,0 +1,24 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +{ + "name": "EDI Storage backend support", + "summary": """ + Base module to allow exchanging files via storage backend (eg: SFTP). + """, + "version": "12.0.1.0.1", + "development_status": "Beta", + "license": "LGPL-3", + "website": "https://github.com/OCA/edi", + "author": "ACSONE,Odoo Community Association (OCA)", + "depends": ["edi_oca", "storage_backend", "component"], + "data": [ + "data/cron.xml", + "data/job_channel_data.xml", + "data/queue_job_function_data.xml", + "security/ir_model_access.xml", + "views/edi_backend_views.xml", + ], + "demo": ["demo/edi_backend_demo.xml"], +} diff --git a/edi_storage_oca/components/__init__.py b/edi_storage_oca/components/__init__.py new file mode 100644 index 0000000000..7bff9ddfcc --- /dev/null +++ b/edi_storage_oca/components/__init__.py @@ -0,0 +1,5 @@ +from . import base +from . import check +from . import send +from . import receive +from . import listener diff --git a/edi_storage_oca/components/base.py b/edi_storage_oca/components/base.py new file mode 100644 index 0000000000..f03cc016b7 --- /dev/null +++ b/edi_storage_oca/components/base.py @@ -0,0 +1,99 @@ +# Copyright 2020 ACSONE +# Copyright 2022 Camptocamp +# @author: Simone Orsi +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). +import logging +from pathlib import PurePath + +from odoo.addons.component.core import AbstractComponent + +_logger = logging.getLogger(__file__) + + +class EDIStorageComponentMixin(AbstractComponent): + + _name = "edi.storage.component.mixin" + _inherit = "edi.component.mixin" + # Components having `_storage_backend_type` will have precedence. + # If the value is not set, generic components will be used. + _storage_backend_type = None + + @classmethod + def _component_match(cls, work, usage=None, model_name=None, **kw): + res = super()._component_match(work, usage=usage, model_name=model_name, **kw) + storage_type = kw.get("storage_backend_type") + if storage_type and cls._storage_backend_type: + return cls._storage_backend_type == storage_type + return res + + @property + def storage(self): + return self.backend.storage_id + + def _dir_by_state(self, direction, state): + """Return remote directory path by direction and state. + + :param direction: string stating direction of the exchange + :param state: string stating state of the exchange + :return: PurePath object + """ + assert direction in ("input", "output") + assert state in ("pending", "done", "error") + return PurePath( + (self.backend[direction + "_dir_" + state] or "").strip().rstrip("/") + ) + + def _remote_file_path(self, direction, state, filename): + """Return remote file path by direction and state for give filename. + + :param direction: string stating direction of the exchange + :param state: string stating state of the exchange + :param filename: string for file name + :return: PurePath object + """ + _logger.warning( + "Call of deprecated function `_remote_file_path`. " + "Please use `_get_remote_file_path` instead.", + ) + return self._dir_by_state(direction, state) / filename.strip("/ ") + + def _get_remote_file_path(self, state, filename=None): + """Retrieve remote path for current exchange record.""" + filename = filename or self.exchange_record.exchange_filename + direction = self.exchange_record.direction + directory = self._dir_by_state(direction, state).as_posix() + path = self.exchange_record.type_id._storage_fullpath( + directory=directory, filename=filename + ) + return path + + def _get_remote_file(self, state, filename=None, binary=False): + """Get file for current exchange_record in the given destination state. + + :param state: string ("pending", "done", "error") + :param filename: custom file name, exchange_record filename used by default + :return: remote file content as string + """ + path = self._get_remote_file_path(state, filename=filename) + try: + # TODO: support match via pattern (eg: filename-prefix-*) + # otherwise is impossible to retrieve input files and acks + # (the date will never match) + return self.storage._get_b64_data(path.as_posix(), binary=binary) + except FileNotFoundError: + _logger.info( + "Ignored FileNotFoundError when trying " + "to get file %s into path %s for state %s", + filename, + path, + state, + ) + return None + except OSError: + _logger.info( + "Ignored OSError when trying to get file %s into path %s for state %s", + filename, + path, + state, + ) + return None diff --git a/edi_storage_oca/components/check.py b/edi_storage_oca/components/check.py new file mode 100644 index 0000000000..2cd97fb4a0 --- /dev/null +++ b/edi_storage_oca/components/check.py @@ -0,0 +1,92 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +import logging + +from odoo.tools import pycompat + +from odoo.addons.component.core import Component + +_logger = logging.getLogger(__name__) + + +class EDIStorageCheckComponentMixin(Component): + + _name = "edi.storage.component.check" + _inherit = [ + "edi.component.check.mixin", + "edi.storage.component.mixin", + ] + _usage = "storage.check" + + def check(self): + return self._exchange_output_check() + + def _exchange_output_check(self): + """Check status output exchange and update record. + + 1. check if the file has been processed already (done) + 2. if yes, post message and exit + 3. if not, check for errors + 4. if no errors, return + + :return: boolean + * False if there's nothing else to be done + * True if file still need action + """ + if self._get_remote_file("done"): + _logger.info( + "%s done", + self.exchange_record.identifier, + ) + if ( + not self.exchange_record.edi_exchange_state + == "output_sent_and_processed" + ): + self.exchange_record.edi_exchange_state = "output_sent_and_processed" + self.exchange_record._notify_done() + return False + + error = self._get_remote_file("error") + if error: + _logger.info( + "%s error", + self.exchange_record.identifier, + ) + # Assume a text file will be placed there w/ the same name and error suffix + err_filename = self.exchange_record.exchange_filename + ".error" + error_report = ( + self._get_remote_file("error", filename=err_filename) or "no-report" + ) + if self.exchange_record.edi_exchange_state == "output_sent": + self.exchange_record.update( + { + "edi_exchange_state": "output_sent_and_error", + "exchange_error": pycompat.to_text(error_report), + } + ) + self.exchange_record._notify_error("process_ko") + return False + return True + + # FIXME: this is not used ATM -> should be refactored + # into an incoming exchange. + # The backend will look for records needing an ack + # and generate and ack record. + def _exchange_output_handle_ack(self): + ack_type = self.exchange_record.type_id.ack_type_id + filename = ack_type._make_exchange_filename(self.exchange_record) + ack_file = self._get_remote_file("done", filename=filename) + if ack_file: + self.backend.create_record( + ack_type.code, + { + "parent_id": self.exchange_record.id, + "exchange_file": ack_file, + "edi_exchange_state": "input_received", + }, + ) + self.exchange_record._notify_ack_received() + else: + self.exchange_record._notify_ack_missing() diff --git a/edi_storage_oca/components/listener.py b/edi_storage_oca/components/listener.py new file mode 100644 index 0000000000..91134d7ce0 --- /dev/null +++ b/edi_storage_oca/components/listener.py @@ -0,0 +1,82 @@ +# Copyright 2021 ForgeFlow S.L. (https://www.forgeflow.com) +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +import functools +from pathlib import PurePath + +from odoo.addons.component.core import Component + + +class EdiStorageListener(Component): + _name = "edi.storage.component.listener" + _inherit = "base.event.listener" + + def _move_file(self, storage, from_dir_str, to_dir_str, filename): + from_dir = PurePath(from_dir_str) + to_dir = PurePath(to_dir_str) + if filename not in storage._list(from_dir.as_posix()): + # The file might have been moved after a previous error. + return False + self._add_after_commit_hook( + storage.move_files, [(from_dir / filename).as_posix()], to_dir.as_posix() + ) + return True + + def _remove_file(self, storage, from_dir_str, filename): + from_dir = PurePath(from_dir_str) + if filename not in storage._list(from_dir.as_posix()): + # The file might have been moved after a previous error. + return False + self._add_after_commit_hook(storage.delete, (from_dir / filename).as_posix()) + return True + + def _add_after_commit_hook(self, partial_func, *args): + """Add hook after commit to move the file when transaction is over.""" + self.env.cr.after( + "commit", + functools.partial(partial_func, *args), + ) + + def on_edi_exchange_done(self, record): + storage = record.backend_id.storage_id + res = False + if record.direction == "input" and storage: + file = record.exchange_filename + pending_dir = record.type_id._storage_fullpath( + record.backend_id.input_dir_pending + ).as_posix() + done_dir = record.type_id._storage_fullpath( + record.backend_id.input_dir_done + ).as_posix() + error_dir = record.type_id._storage_fullpath( + record.backend_id.input_dir_error + ).as_posix() + if record.backend_id.input_dir_remove: + res = self._remove_file(storage, pending_dir, file) + if not res: + res = self._remove_file(storage, error_dir, file) + return res + if not done_dir: + return res + res = self._move_file(storage, pending_dir, done_dir, file) + if not res: + # If a file previously failed it should have been previously + # moved to the error dir, therefore it is not present in the + # pending dir and we need to retry from error dir. + res = self._move_file(storage, error_dir, done_dir, file) + return res + + def on_edi_exchange_error(self, record): + storage = record.backend_id.storage_id + res = False + if record.direction == "input" and storage: + file = record.exchange_filename + pending_dir = record.type_id._storage_fullpath( + record.backend_id.input_dir_pending + ).as_posix() + error_dir = record.type_id._storage_fullpath( + record.backend_id.input_dir_error + ).as_posix() + if error_dir: + res = self._move_file(storage, pending_dir, error_dir, file) + return res diff --git a/edi_storage_oca/components/receive.py b/edi_storage_oca/components/receive.py new file mode 100644 index 0000000000..aefb66642f --- /dev/null +++ b/edi_storage_oca/components/receive.py @@ -0,0 +1,19 @@ +# Copyright 2021 ForgeFlow S.L. (https://www.forgeflow.com) +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +from odoo.addons.component.core import Component + + +class EDIStorageReceiveComponent(Component): + + _name = "edi.storage.component.receive" + _inherit = [ + "edi.component.receive.mixin", + "edi.storage.component.mixin", + ] + _usage = "storage.receive" + + def receive(self): + path = self._get_remote_file_path("pending") + filedata = self.storage._get_b64_data(path.as_posix()) + return filedata diff --git a/edi_storage_oca/components/send.py b/edi_storage_oca/components/send.py new file mode 100644 index 0000000000..9267f6847d --- /dev/null +++ b/edi_storage_oca/components/send.py @@ -0,0 +1,36 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +from odoo.addons.component.core import Component + + +class EDIStorageSendComponent(Component): + + _name = "edi.storage.component.send" + _inherit = [ + "edi.component.send.mixin", + "edi.storage.component.mixin", + ] + _usage = "storage.send" + + def send(self): + # If the file has been sent already, refresh its state + # TODO: double check if this is useless + # since the backend checks the state already + checker = self.component(usage="storage.check") + result = checker.check() + if not result: + # all good here + return True + filedata = self.exchange_record.exchange_file + path = self._get_remote_file_path("pending") + self.storage._add_b64_data(path.as_posix(), filedata, binary=False) + # TODO: delegate this to generic storage backend + # except paramiko.ssh_exception.AuthenticationException: + # # TODO this exc handling should be moved to sftp backend IMO + # error = _("Authentication error") + # state = "error_on_send" + # TODO: catch other specific exceptions + # this will swallow all the exceptions! + return True diff --git a/edi_storage_oca/data/cron.xml b/edi_storage_oca/data/cron.xml new file mode 100644 index 0000000000..df21892a08 --- /dev/null +++ b/edi_storage_oca/data/cron.xml @@ -0,0 +1,17 @@ + + + + EDI backend storage check pending input + + + 1 + hours + -1 + + + code + model.search([('storage_id', '!=', False)])._storage_cron_check_pending_input() + + diff --git a/edi_storage_oca/data/job_channel_data.xml b/edi_storage_oca/data/job_channel_data.xml new file mode 100644 index 0000000000..b3b3b770d7 --- /dev/null +++ b/edi_storage_oca/data/job_channel_data.xml @@ -0,0 +1,6 @@ + + + edi_storage + + + diff --git a/edi_storage_oca/data/queue_job_function_data.xml b/edi_storage_oca/data/queue_job_function_data.xml new file mode 100644 index 0000000000..1965284193 --- /dev/null +++ b/edi_storage_oca/data/queue_job_function_data.xml @@ -0,0 +1,7 @@ + + + + _storage_create_record_if_missing + + + diff --git a/edi_storage_oca/demo/edi_backend_demo.xml b/edi_storage_oca/demo/edi_backend_demo.xml new file mode 100644 index 0000000000..5b003460f7 --- /dev/null +++ b/edi_storage_oca/demo/edi_backend_demo.xml @@ -0,0 +1,14 @@ + + + + Storage Demo EDI backend + + + demo_in/pending + demo_in/done + demo_in/error + demo_out/pending + demo_out/done + demo_out/error + + diff --git a/edi_storage_oca/i18n/edi_storage.pot b/edi_storage_oca/i18n/edi_storage.pot new file mode 100644 index 0000000000..d6944e83bd --- /dev/null +++ b/edi_storage_oca/i18n/edi_storage.pot @@ -0,0 +1,77 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * edi_storage +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 13.0\n" +"Report-Msgid-Bugs-To: \n" +"Last-Translator: \n" +"Language-Team: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: \n" + +#. module: edi_storage +#: model:ir.model,name:edi_storage.model_edi_backend +msgid "EDI Backend" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__input_dir_done +msgid "Input done directory" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__input_dir_error +msgid "Input error directory" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__input_dir_pending +msgid "Input pending directory" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__output_dir_done +msgid "Output done directory" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__output_dir_error +msgid "Output error directory" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__output_dir_pending +msgid "Output pending directory" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,help:edi_storage.field_edi_backend__input_dir_done +#: model:ir.model.fields,help:edi_storage.field_edi_backend__output_dir_done +msgid "Path to folder for doneful operations" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,help:edi_storage.field_edi_backend__input_dir_error +#: model:ir.model.fields,help:edi_storage.field_edi_backend__output_dir_error +msgid "Path to folder for error operations" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,help:edi_storage.field_edi_backend__input_dir_pending +#: model:ir.model.fields,help:edi_storage.field_edi_backend__output_dir_pending +msgid "Path to folder for pending operations" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,field_description:edi_storage.field_edi_backend__storage_id +msgid "Storage backend" +msgstr "" + +#. module: edi_storage +#: model:ir.model.fields,help:edi_storage.field_edi_backend__storage_id +msgid "Storage for in-out files" +msgstr "" diff --git a/edi_storage_oca/i18n/edi_storage_oca.pot b/edi_storage_oca/i18n/edi_storage_oca.pot new file mode 100644 index 0000000000..f012629aec --- /dev/null +++ b/edi_storage_oca/i18n/edi_storage_oca.pot @@ -0,0 +1,129 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * edi_storage_oca +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 14.0\n" +"Report-Msgid-Bugs-To: \n" +"Last-Translator: \n" +"Language-Team: \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: \n" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__display_name +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type__display_name +msgid "Display Name" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model,name:edi_storage_oca.model_edi_backend +msgid "EDI Backend" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model,name:edi_storage_oca.model_edi_exchange_type +msgid "EDI Exchange Type" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.actions.server,name:edi_storage_oca.cron_check_storage_pending_input_ir_actions_server +#: model:ir.cron,cron_name:edi_storage_oca.cron_check_storage_pending_input +#: model:ir.cron,name:edi_storage_oca.cron_check_storage_pending_input +msgid "EDI backend storage check pending input" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type__exchange_filename_pattern +msgid "Exchange Filename Pattern" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_exchange_type__exchange_filename_pattern +msgid "" +"For output exchange types this should be a formatting string with the following variables available (to be used between brackets, `{}`): `exchange_record`, `record_name`, `type` and `dt`. For instance, a valid string would be {record_name}-{type.code}-{dt}\n" +"For input exchange types related to storage backends it should be a regex expression to filter the files to be fetched from the pending directory in the related storage. E.g: `.*my-type-[0-9]*.\\.csv`" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__id +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type__id +msgid "ID" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_done +msgid "Input done directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_error +msgid "Input error directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_pending +msgid "Input pending directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend____last_update +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type____last_update +msgid "Last Modified on" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__output_dir_done +msgid "Output done directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__output_dir_error +msgid "Output error directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__output_dir_pending +msgid "Output pending directory" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__input_dir_done +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__output_dir_done +msgid "Path to folder for doneful operations" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__input_dir_error +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__output_dir_error +msgid "Path to folder for error operations" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__input_dir_pending +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__output_dir_pending +msgid "Path to folder for pending operations" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_remove +msgid "Remove input after done" +msgstr "" + +#. module: edi_storage_oca +#: model_terms:ir.ui.view,arch_db:edi_storage_oca.edi_backend_view_form +msgid "Storage" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__storage_id +msgid "Storage backend" +msgstr "" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__storage_id +msgid "Storage for in-out files" +msgstr "" diff --git a/edi_storage_oca/i18n/es.po b/edi_storage_oca/i18n/es.po new file mode 100644 index 0000000000..f7c0aed273 --- /dev/null +++ b/edi_storage_oca/i18n/es.po @@ -0,0 +1,140 @@ +# Translation of Odoo Server. +# This file contains the translation of the following modules: +# * edi_storage_oca +# +msgid "" +msgstr "" +"Project-Id-Version: Odoo Server 14.0\n" +"Report-Msgid-Bugs-To: \n" +"PO-Revision-Date: 2023-11-20 22:34+0000\n" +"Last-Translator: Ivorra78 \n" +"Language-Team: none\n" +"Language: es\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: \n" +"Plural-Forms: nplurals=2; plural=n != 1;\n" +"X-Generator: Weblate 4.17\n" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__display_name +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type__display_name +msgid "Display Name" +msgstr "Mostrar Nombre" + +#. module: edi_storage_oca +#: model:ir.model,name:edi_storage_oca.model_edi_backend +msgid "EDI Backend" +msgstr "Servidor EDI" + +#. module: edi_storage_oca +#: model:ir.model,name:edi_storage_oca.model_edi_exchange_type +msgid "EDI Exchange Type" +msgstr "Tipo de Intercambio EDI" + +#. module: edi_storage_oca +#: model:ir.actions.server,name:edi_storage_oca.cron_check_storage_pending_input_ir_actions_server +#: model:ir.cron,cron_name:edi_storage_oca.cron_check_storage_pending_input +#: model:ir.cron,name:edi_storage_oca.cron_check_storage_pending_input +msgid "EDI backend storage check pending input" +msgstr "Comprobación de entrada pendiente de almacenamiento del servidor EDI" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type__exchange_filename_pattern +msgid "Exchange Filename Pattern" +msgstr "Patrón de Intercambio de Archivos" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_exchange_type__exchange_filename_pattern +msgid "" +"For output exchange types this should be a formatting string with the following variables available (to be used between brackets, `{}`): `exchange_record`, `record_name`, `type` and `dt`. For instance, a valid string would be {record_name}-{type.code}-{dt}\n" +"For input exchange types related to storage backends it should be a regex expression to filter the files to be fetched from the pending directory in the related storage. E.g: `.*my-type-[0-9]*.\\.csv`" +msgstr "" +"Para los tipos de intercambio de salida, esto debe ser una cadena de formato " +"con las siguientes variables disponibles (que se usarán entre corchetes, " +"`{}`): `exchange_record`, `record_name`, `type` y `dt`. Por ejemplo, una " +"cadena válida sería {record_name}-{type.code}-{dt}\n" +"Para los tipos de intercambio de entrada relacionados con backends de " +"almacenamiento, debe ser una expresión regular para filtrar los archivos que " +"se recuperarán del directorio pendiente en el almacenamiento relacionado. " +"Por ejemplo: `.*mi-tipo-[0-9]*.\\.csv`" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__id +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type__id +msgid "ID" +msgstr "ID" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_done +msgid "Input done directory" +msgstr "Entrada directorio realizada" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_error +msgid "Input error directory" +msgstr "Directorio de error de entrada" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_pending +msgid "Input pending directory" +msgstr "Entrada directorio pendiente" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend____last_update +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_exchange_type____last_update +msgid "Last Modified on" +msgstr "Última Modificación el" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__output_dir_done +msgid "Output done directory" +msgstr "Salida directorio realizada" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__output_dir_error +msgid "Output error directory" +msgstr "Directorio de error de salida" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__output_dir_pending +msgid "Output pending directory" +msgstr "Salida directorio pendiente" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__input_dir_done +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__output_dir_done +msgid "Path to folder for doneful operations" +msgstr "Ruta de acceso a la carpeta para las operaciones realizadas" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__input_dir_error +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__output_dir_error +msgid "Path to folder for error operations" +msgstr "Ruta a la carpeta para operaciones de error" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__input_dir_pending +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__output_dir_pending +msgid "Path to folder for pending operations" +msgstr "Ruta a la carpeta de operaciones pendientes" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__input_dir_remove +msgid "Remove input after done" +msgstr "Eliminar la entrada una vez realizada" + +#. module: edi_storage_oca +#: model_terms:ir.ui.view,arch_db:edi_storage_oca.edi_backend_view_form +msgid "Storage" +msgstr "Almacenamiento" + +#. module: edi_storage_oca +#: model:ir.model.fields,field_description:edi_storage_oca.field_edi_backend__storage_id +msgid "Storage backend" +msgstr "Servidor de almacenamiento" + +#. module: edi_storage_oca +#: model:ir.model.fields,help:edi_storage_oca.field_edi_backend__storage_id +msgid "Storage for in-out files" +msgstr "Almacenamiento de archivos de entrada-salida" diff --git a/edi_storage_oca/models/__init__.py b/edi_storage_oca/models/__init__.py new file mode 100644 index 0000000000..f34a72163f --- /dev/null +++ b/edi_storage_oca/models/__init__.py @@ -0,0 +1,2 @@ +from . import edi_backend +from . import edi_exchange_type diff --git a/edi_storage_oca/models/edi_backend.py b/edi_storage_oca/models/edi_backend.py new file mode 100644 index 0000000000..d9ff6c4552 --- /dev/null +++ b/edi_storage_oca/models/edi_backend.py @@ -0,0 +1,179 @@ +# Copyright 2020 ACSONE SA +# @author Simone Orsi +# Copyright 2021 ForgeFlow S.L. (https://www.forgeflow.com) +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +import logging +import os + +from odoo import fields, models + +_logger = logging.getLogger(__name__) + + +class EDIBackend(models.Model): + + _inherit = "edi.backend" + + storage_id = fields.Many2one( + string="Storage backend", + comodel_name="storage.backend", + help="Storage for in-out files", + ondelete="restrict", + ) + """ + We assume the exchanges happen it 2 ways (input, output) + and we have a hierarchy of directory like: + + from_A_to_B + |- pending + |- done + |- error + from_B_to_A + |- pending + |- done + |- error + + where A and B are the partners exchanging data and they are in turn + sender and receiver and vice versa. + """ + # TODO: these paths should probably be by type instead + # Here we can maybe set a common root folder for this exchange. + input_dir_pending = fields.Char( + "Input pending directory", help="Path to folder for pending operations" + ) + input_dir_remove = fields.Boolean("Remove input after done") + input_dir_done = fields.Char( + "Input done directory", help="Path to folder for doneful operations" + ) + input_dir_error = fields.Char( + "Input error directory", help="Path to folder for error operations" + ) + output_dir_pending = fields.Char( + "Output pending directory", help="Path to folder for pending operations" + ) + output_dir_done = fields.Char( + "Output done directory", help="Path to folder for doneful operations" + ) + output_dir_error = fields.Char( + "Output error directory", help="Path to folder for error operations" + ) + + _storage_actions = ("check", "send", "receive") + + def _get_component_usage_candidates(self, exchange_record, key): + candidates = super()._get_component_usage_candidates(exchange_record, key) + if not self.storage_id or key not in self._storage_actions: + return candidates + return ["storage.{}".format(key)] + candidates + + def _component_match_attrs(self, exchange_record, key): + # Override to inject storage_backend_type + res = super()._component_match_attrs(exchange_record, key) + if not self.storage_id or key not in self._storage_actions: + return res + res["storage_backend_type"] = self.storage_id.backend_type + return res + + def _component_sort_key(self, component_class): + res = super()._component_sort_key(component_class) + # Override to give precedence by storage_backend_type when needed. + if not self.storage_id: + return res + return ( + 1 if getattr(component_class, "_storage_backend_type", False) else 0, + ) + res + + def _storage_cron_check_pending_input(self, **kw): + for backend in self: + backend._storage_check_pending_input(**kw) + + def _storage_check_pending_input(self, **kw): + """Create new exchange records if new files found. + + Collect input exchange types and for each of them, + check by pattern if the a new exchange record is required. + """ + self.ensure_one() + if not self.storage_id or not self.input_dir_pending: + _logger.info( + "%s ignored: no storage and/or input directory specified.", self.name + ) + return False + + exchange_types = self.env["edi.exchange.type"].search( + self._storage_exchange_type_pending_input_domain() + ) + for exchange_type in exchange_types: + # NOTE: this call might keep hanging the cron + # if the remote storage is slow (eg: too many files) + # We should probably run this code in a separate job per exchange type. + file_names = self._storage_get_input_filenames(exchange_type) + _logger.info( + "Processing exchange type '%s': found %s files to process", + exchange_type.display_name, + len(file_names), + ) + for file_name in file_names: + self.with_delay()._storage_create_record_if_missing( + exchange_type, file_name + ) + return True + + def _storage_exchange_type_pending_input_domain(self): + """Domain for retrieving input exchange types.""" + return [ + ("backend_type_id", "=", self.backend_type_id.id), + ("direction", "=", "input"), + "|", + ("backend_id", "=", False), + ("backend_id", "=", self.id), + ] + + def _storage_create_record_if_missing(self, exchange_type, remote_file_name): + """Create a new exchange record for given type and file name if missing.""" + file_name = os.path.basename(remote_file_name) + extra_domain = [("exchange_filename", "=", file_name)] + existing = self._find_existing_exchange_records( + exchange_type, extra_domain=extra_domain, count_only=True + ) + if existing: + return + record = self.create_record( + exchange_type.code, + { + "edi_exchange_state": "input_pending" + } + ) + record.exchange_filename = file_name + _logger.debug("%s: new exchange record generated.", self.name) + return record.identifier + + def _storage_get_input_filenames(self, exchange_type): + full_input_dir_pending = exchange_type._storage_fullpath( + self.input_dir_pending + ).as_posix() + if not exchange_type.exchange_filename_pattern: + # If there is not pattern, return everything + filenames = [ + x + for x in self.storage_id._list(full_input_dir_pending) + if x.strip("/") + ] + return filenames + + bits = [exchange_type.exchange_filename_pattern] + if exchange_type.exchange_file_ext: + bits.append(r"\." + exchange_type.exchange_file_ext) + pattern = "".join(bits) + full_paths = self.storage_id._find_files(pattern, full_input_dir_pending) + pending_path_len = len(full_input_dir_pending) + return [p[pending_path_len:].strip("/") for p in full_paths] + + def _check_output_exchange_sync(self, **kw): + # Do not skip sent records when dealing w/ storage related exchanges, + # because we want to update the file state + # depending on where they are in the external folder. + if self.storage_id: + kw["skip_sent"] = False + return super()._check_output_exchange_sync(**kw) diff --git a/edi_storage_oca/models/edi_exchange_type.py b/edi_storage_oca/models/edi_exchange_type.py new file mode 100644 index 0000000000..258576edc3 --- /dev/null +++ b/edi_storage_oca/models/edi_exchange_type.py @@ -0,0 +1,62 @@ +# Copyright 2021 ForgeFlow S.L. (https://www.forgeflow.com) +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +from pathlib import PurePath + +from odoo import fields, models + + +class EDIExchangeType(models.Model): + _inherit = "edi.exchange.type" + + # Extend help to explain new usage. + exchange_filename_pattern = fields.Char( + help="For output exchange types this should be a formatting string " + "with the following variables available (to be used between " + "brackets, `{}`): `exchange_record`, `record_name`, `type` and " + "`dt`. For instance, a valid string would be " + "{record_name}-{type.code}-{dt}\n" + "For input exchange types related to storage backends " + "it should be a regex expression to filter " + "the files to be fetched from the pending directory in the related " + "storage. E.g: `.*my-type-[0-9]*.\\.csv`" + ) + + def _storage_path(self): + """Retrieve specific path for current exchange type. + + In your exchange type you can pass this config: + + storage: + # simple string + path: path/to/file + + Or + + storage: + # name of the param containing the path + path_config_param: path/to/file + + Thanks to the param you could even configure it by env. + """ + self.ensure_one() + storage_settings = self.advanced_settings.get("storage", {}) + path = storage_settings.get("path") + if path: + return PurePath(path) + path_config_param = storage_settings.get("path_config_param") + if path_config_param: + icp = self.env["ir.config_parameter"].sudo() + path = icp.get_param(path_config_param) + if path: + return PurePath(path) + + def _storage_fullpath(self, directory=None, filename=None): + self.ensure_one() + path_prefix = self._storage_path() + path = PurePath((directory or "").strip().rstrip("/")) + if path_prefix: + path = path_prefix / path + if filename: + path = path / filename.strip("/") + return path diff --git a/edi_storage_oca/readme/CONTRIBUTORS.rst b/edi_storage_oca/readme/CONTRIBUTORS.rst new file mode 100644 index 0000000000..7f7caf8d54 --- /dev/null +++ b/edi_storage_oca/readme/CONTRIBUTORS.rst @@ -0,0 +1,7 @@ +* Simone Orsi +* Foram Shah +* Lois Rilo +* `Trobz `_: + + * Thien + diff --git a/edi_storage_oca/readme/CREDITS.rst b/edi_storage_oca/readme/CREDITS.rst new file mode 100644 index 0000000000..b15392d601 --- /dev/null +++ b/edi_storage_oca/readme/CREDITS.rst @@ -0,0 +1 @@ +The backport of this module from 14.0 to 12.0 was financially supported by Camptocamp diff --git a/edi_storage_oca/readme/DESCRIPTION.rst b/edi_storage_oca/readme/DESCRIPTION.rst new file mode 100644 index 0000000000..bdef14a8ad --- /dev/null +++ b/edi_storage_oca/readme/DESCRIPTION.rst @@ -0,0 +1,23 @@ +Allow exchange files using storage backends from `OCA/storage`. + +This module adds a storage backend relation on the EDI backend. +There you can configure the backend to be used (most often and SFTP) +and the paths where to read or put files. + +Often the convention when exchanging files via SFTP +is to have one input forder (to receive files) +and an output folder (to send files). + +Inside this folder you have this hierarchy:: + + input/output folder + |- pending + |- done + |- error + +* `pending` folder contains files that have been just sent +* `done` folder contains files that have been processes successfully +* `error` folder contains files with errors and cannot be processed + +The storage handlers take care of reading files and putting files +in/from the right place and update exchange records data accordingly. diff --git a/edi_storage_oca/readme/USAGE.rst b/edi_storage_oca/readme/USAGE.rst new file mode 100644 index 0000000000..a47c64c841 --- /dev/null +++ b/edi_storage_oca/readme/USAGE.rst @@ -0,0 +1 @@ +Go to "EDI -> EDI backend" then configure your backend to use a storage backend. diff --git a/edi_storage_oca/security/ir_model_access.xml b/edi_storage_oca/security/ir_model_access.xml new file mode 100644 index 0000000000..b7d7b84987 --- /dev/null +++ b/edi_storage_oca/security/ir_model_access.xml @@ -0,0 +1,12 @@ + + + + access_storage_backend EDI manager + + + + + + + + diff --git a/edi_storage_oca/static/description/icon.png b/edi_storage_oca/static/description/icon.png new file mode 100644 index 0000000000..3a0328b516 Binary files /dev/null and b/edi_storage_oca/static/description/icon.png differ diff --git a/edi_storage_oca/static/description/index.html b/edi_storage_oca/static/description/index.html new file mode 100644 index 0000000000..a4e3077ed3 --- /dev/null +++ b/edi_storage_oca/static/description/index.html @@ -0,0 +1,463 @@ + + + + + + +EDI Storage backend support + + + +
+

EDI Storage backend support

+ + +

Beta License: LGPL-3 OCA/edi Translate me on Weblate Try me on Runboat

+

Allow exchange files using storage backends from OCA/storage.

+

This module adds a storage backend relation on the EDI backend. +There you can configure the backend to be used (most often and SFTP) +and the paths where to read or put files.

+

Often the convention when exchanging files via SFTP +is to have one input forder (to receive files) +and an output folder (to send files).

+

Inside this folder you have this hierarchy:

+
+input/output folder
+    |- pending
+    |- done
+    |- error
+
+
    +
  • pending folder contains files that have been just sent
    • +
  • done folder contains files that have been processes successfully
    • +
  • error folder contains files with errors and cannot be processed
    • +
+

The storage handlers take care of reading files and putting files +in/from the right place and update exchange records data accordingly.

+

Table of contents

+ +
+

Usage

+

Go to “EDI -> EDI backend” then configure your backend to use a storage backend.

+
+
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+
+

Credits

+
+

Authors

+
    +
  • ACSONE
    • +
+
+
+

Contributors

+ +
+
+

Other credits

+

The backport of this module from 14.0 to 12.0 was financially supported by Camptocamp

+
+
+

Maintainers

+

This module is maintained by the OCA.

+Odoo Community Association +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

This module is part of the OCA/edi project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/edi_storage_oca/tests/__init__.py b/edi_storage_oca/tests/__init__.py new file mode 100644 index 0000000000..18bc953002 --- /dev/null +++ b/edi_storage_oca/tests/__init__.py @@ -0,0 +1,5 @@ +from . import test_edi_backend_storage +from . import test_components_base +from . import test_component_match +from . import test_edi_storage_listener +from . import test_exchange_type diff --git a/edi_storage_oca/tests/common.py b/edi_storage_oca/tests/common.py new file mode 100644 index 0000000000..03304672c5 --- /dev/null +++ b/edi_storage_oca/tests/common.py @@ -0,0 +1,171 @@ +# Copyright 2020 ACSONE SA/NV () +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html). +import base64 +import functools + +import mock + +from odoo.addons.edi_oca.tests.common import EDIBackendCommonComponentTestCase + +STORAGE_BACKEND_MOCK_PATH = ( + "odoo.addons.storage_backend.models.storage_backend.StorageBackend" +) + + +class TestEDIStorageBase(EDIBackendCommonComponentTestCase): + @classmethod + def _get_backend(cls): + return cls.env.ref("edi_storage_oca.demo_edi_backend_storage") + + @classmethod + def _setup_records(cls): + super()._setup_records() + cls.filedata = base64.b64encode(b"This is a simple file") + vals = { + "model": cls.partner._name, + "res_id": cls.partner.id, + "exchange_file": cls.filedata, + } + cls.record = cls.backend.create_record("test_csv_output", vals) + cls.record_input = cls.backend.create_record("test_csv_input", vals) + + cls.fakepath = "/tmp/{}".format(cls._filename(cls)) + with open(cls.fakepath, "w+b") as fakefile: + fakefile.write(b"filecontent") + + cls.fakepath_ack = "/tmp/{}.ack".format(cls._filename(cls)) + with open(cls.fakepath_ack, "w+b") as fakefile: + fakefile.write(b"ACK filecontent") + + cls.fakepath_error = "/tmp/{}.error".format(cls._filename(cls)) + with open(cls.fakepath_error, "w+b") as fakefile: + fakefile.write(b"ERROR XYZ: line 2 broken on bla bla") + + cls.fakepath_input_pending_1 = "/tmp/test-input-001.csv" + with open(cls.fakepath_input_pending_1, "w+b") as fakefile: + fakefile.write(b"I received this in my storage.") + + cls.fakepath_input_pending_2 = "/tmp/test-input-002.csv" + with open(cls.fakepath_input_pending_2, "w+b") as fakefile: + fakefile.write(b"I received that in my storage.") + + cls.checker = cls.backend._find_component( + cls.partner._name, + ["storage.check"], + work_ctx={"exchange_record": cls.record}, + ) + cls.checker_input = cls.backend._find_component( + cls.partner._name, + ["storage.check"], + work_ctx={"exchange_record": cls.record_input}, + ) + cls.sender = cls.backend._find_component( + cls.partner._name, + ["storage.send"], + work_ctx={"exchange_record": cls.record}, + ) + + def setUp(self): + super().setUp() + self._storage_backend_calls = [] + + def _filename(self, record=None, ack=False): + record = record or self.record + if ack: + record.type_id.ack_type_id._make_exchange_filename(record) + return record.exchange_filename + + def _file_fullpath(self, state, record=None, ack=False, fname=None, checker=None): + record = record or self.record + checker = checker or self.checker + if not fname: + fname = self._filename(record, ack=ack) + if state == "error-report": + # Exception as we read from the same path but w/ error suffix + state = "error" + fname += ".error" + return checker._get_remote_file_path(state, filename=fname).as_posix() + + def _mocked_backend_get(self, mocked_paths, path, **kwargs): + self._storage_backend_calls.append(path) + if mocked_paths.get(path): + with open(mocked_paths.get(path), "rb") as remote_file: + return remote_file.read() + raise FileNotFoundError() + + def _mocked_backend_add(self, path, data, **kwargs): + self._storage_backend_calls.append(path) + + def _mocked_backend_list_files(self, mocked_paths, path, **kwargs): + files = [] + path_length = len(path) + for p in mocked_paths.keys(): + if path in p and path != p: + files.append(p[path_length:]) + return files + + def _mock_storage_backend_get(self, mocked_paths): + mocked = functools.partial(self._mocked_backend_get, mocked_paths) + return mock.patch(STORAGE_BACKEND_MOCK_PATH + "._get_b64_data", mocked) + + def _mock_storage_backend_add(self): + return mock.patch( + STORAGE_BACKEND_MOCK_PATH + "._add_b64_data", self._mocked_backend_add + ) + + def _mock_storage_backend_list_files(self, mocked_paths): + mocked = functools.partial(self._mocked_backend_list_files, mocked_paths) + return mock.patch(STORAGE_BACKEND_MOCK_PATH + "._list", mocked) + + def _mock_storage_backend_find_files(self, result): + def _result(self, pattern, relative_path=None, **kw): + return result + + return mock.patch(STORAGE_BACKEND_MOCK_PATH + "._find_files", _result) + + def _test_result( + self, + record, + expected_values, + expected_messages=None, + state_paths=None, + ): + state_paths = state_paths or ("done", "pending", "error") + # Paths will be something like: + # [ + # 'demo_out/pending/$filename.csv', + # 'demo_out/pending/$filename.csv', + # 'demo_out/error/$filename.csv', + # ] + for state in state_paths: + path = self._file_fullpath(state, record=record) + self.assertIn(path, self._storage_backend_calls) + self.assertRecordValues(record, [expected_values]) + if expected_messages: + # consider only edi related messages + messages = record.record.message_ids.filtered( + lambda x: "edi-exchange" in x.body + ) + self.assertEqual(len(messages), len(expected_messages)) + for msg_rec, expected in zip(messages, expected_messages): + self.assertIn(expected["message"], msg_rec.body) + self.assertIn("level-" + expected["level"], msg_rec.body) + # TODO: test content of file sent + + def _test_send(self, record, mocked_paths=None): + with self._mock_storage_backend_add(): + if mocked_paths: + with self._mock_storage_backend_get(mocked_paths): + self.backend.exchange_send(record) + else: + self.backend.exchange_send(record) + + def _test_run_cron(self, mocked_paths): + with self._mock_storage_backend_add(): + with self._mock_storage_backend_get(mocked_paths): + self.backend._cron_check_output_exchange_sync() + + def _test_run_cron_pending_input(self, mocked_paths): + with self._mock_storage_backend_add(): + with self._mock_storage_backend_list_files(mocked_paths): + self.backend._storage_cron_check_pending_input() diff --git a/edi_storage_oca/tests/test_component_match.py b/edi_storage_oca/tests/test_component_match.py new file mode 100644 index 0000000000..f0a776ae90 --- /dev/null +++ b/edi_storage_oca/tests/test_component_match.py @@ -0,0 +1,93 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +from odoo.addons.component.core import Component +from odoo.addons.edi_oca.tests.common import EDIBackendCommonComponentRegistryTestCase + + +class EDIBackendTestCase(EDIBackendCommonComponentRegistryTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls._load_module_components(cls, "edi_storage_oca") + + @classmethod + def _get_backend(cls): + return cls.env.ref("edi_storage_oca.demo_edi_backend_storage") + + def test_component_match(self): + """Lookup with special match method.""" + + class SFTPCheck(Component): + _name = "sftp.check" + _inherit = "edi.storage.component.check" + _usage = "storage.check" + # _backend_type = "demo_backend" + _storage_backend_type = "sftp" + + class SFTPSend(Component): + _name = "sftp.send" + _inherit = "edi.storage.component.send" + _usage = "storage.send" + # _backend_type = "demo_backend" + _storage_backend_type = "sftp" + + class S3Check(Component): + _name = "s3.check" + _inherit = "edi.storage.component.check" + _usage = "storage.check" + # _exchange_type = "test_csv_output" + _storage_backend_type = "s3" + + class S3Send(Component): + _name = "s3.send" + _inherit = "edi.storage.component.send" + _usage = "storage.send" + # _exchange_type = "test_csv_output" + _storage_backend_type = "s3" + + self._build_components(SFTPCheck, SFTPSend, S3Check, S3Send) + + # Record not relevant for these tests + work_ctx = {"exchange_record": self.env["edi.exchange.record"].browse()} + + component = self.backend._find_component( + "res.partner", + ["storage.check"], + work_ctx=work_ctx, + backend_type="demo_backend", + exchange_type="test_csv_output", + storage_backend_type="s3", + ) + self.assertEqual(component._name, S3Check._name) + + component = self.backend._find_component( + "res.partner", + ["storage.check"], + work_ctx=work_ctx, + backend_type="demo_backend", + exchange_type="test_csv_output", + storage_backend_type="sftp", + ) + self.assertEqual(component._name, SFTPCheck._name) + + component = self.backend._find_component( + "res.partner", + ["storage.send"], + work_ctx=work_ctx, + backend_type="demo_backend", + exchange_type="test_csv_output", + storage_backend_type="sftp", + ) + self.assertEqual(component._name, SFTPSend._name) + + component = self.backend._find_component( + "res.partner", + ["storage.send"], + work_ctx=work_ctx, + backend_type="demo_backend", + exchange_type="test_csv_output", + storage_backend_type="s3", + ) + self.assertEqual(component._name, S3Send._name) diff --git a/edi_storage_oca/tests/test_components_base.py b/edi_storage_oca/tests/test_components_base.py new file mode 100644 index 0000000000..3d3d9e07c8 --- /dev/null +++ b/edi_storage_oca/tests/test_components_base.py @@ -0,0 +1,45 @@ +# Copyright 2020 ACSONE +# @author: Simone Orsi +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +import mock + +from .common import STORAGE_BACKEND_MOCK_PATH, TestEDIStorageBase + + +class EDIStorageComponentTestCase(TestEDIStorageBase): + def test_remote_file_path(self): + to_test = ( + (("input", "pending", "foo.csv"), "demo_in/pending/foo.csv"), + (("input", "done", "foo.csv"), "demo_in/done/foo.csv"), + (("input", "error", "foo.csv"), "demo_in/error/foo.csv"), + (("output", "pending", "foo.csv"), "demo_out/pending/foo.csv"), + (("output", "done", "foo.csv"), "demo_out/done/foo.csv"), + (("output", "error", "foo.csv"), "demo_out/error/foo.csv"), + ) + for _args, expected in to_test: + direction, state, filename = _args + if direction == "input": + checker = self.checker_input + else: + checker = self.checker + path_obj = checker._get_remote_file_path(state, filename) + self.assertEqual(path_obj.as_posix(), expected) + + with self.assertRaises(AssertionError): + self.checker_input._get_remote_file_path("WHATEVER", "foo.csv") + + def test_get_remote_file(self): + with mock.patch(STORAGE_BACKEND_MOCK_PATH + "._get_b64_data") as mocked: + self.checker._get_remote_file("pending") + mocked.assert_called_with( + "demo_out/pending/{}".format(self._filename(self.record)), binary=False + ) + self.checker._get_remote_file("done") + mocked.assert_called_with( + "demo_out/done/{}".format(self._filename(self.record)), binary=False + ) + self.checker._get_remote_file("error") + mocked.assert_called_with( + "demo_out/error/{}".format(self._filename(self.record)), binary=False + ) diff --git a/edi_storage_oca/tests/test_edi_backend_storage.py b/edi_storage_oca/tests/test_edi_backend_storage.py new file mode 100644 index 0000000000..9f52acdd5c --- /dev/null +++ b/edi_storage_oca/tests/test_edi_backend_storage.py @@ -0,0 +1,245 @@ +# Copyright 2020 ACSONE SA/NV () +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl.html). +from freezegun import freeze_time + +from odoo.tools import mute_logger + +from .common import TestEDIStorageBase + +LOGGERS = ( + "odoo.addons.edi_storage_oca.components.check", + "odoo.addons.edi_oca.models.edi_backend", +) + + +@freeze_time("2020-10-21 10:30:00") +class TestEDIBackendOutput(TestEDIStorageBase): + @mute_logger(*LOGGERS) + def test_export_file_sent(self): + """Send, no errors.""" + self.record.edi_exchange_state = "output_pending" + mocked_paths = {self._file_fullpath("pending"): self.fakepath} + # TODO: test send only w/out cron (make sure check works) + # self._test_send(self.record, mocked_paths=mocked_paths) + self._test_run_cron(mocked_paths) + self._test_result( + self.record, + {"edi_exchange_state": "output_sent"}, + expected_messages=[ + { + "message": self.record._exchange_status_message("send_ok"), + "level": "info", + } + ], + ) + + @mute_logger(*LOGGERS) + def test_export_file_already_done(self): + """Already sent, successfully.""" + self.record.edi_exchange_state = "output_sent" + mocked_paths = {self._file_fullpath("done"): self.fakepath} + # TODO: test send only w/out cron (make sure check works) + self._test_run_cron(mocked_paths) + # As we simulate to find a file in `done` folder, + # we should get the final good state + # and only one call to ftp + self._test_result( + self.record, + {"edi_exchange_state": "output_sent_and_processed"}, + state_paths=("done",), + expected_messages=[ + { + "message": self.record._exchange_status_message("process_ok"), + "level": "info", + } + ], + ) + + # FIXME: ack should be handle as an incoming record (new machinery to be added) + # @mute_logger(*LOGGERS) + # def test_export_file_already_done_ack_needed_not_found(self): + # self.record.edi_exchange_state = "output_sent" + # self.record.type_id.ack_needed = True + # mocked_paths = { + # self._file_fullpath("done"): self.fakepath, + # } + # self._test_run_cron(mocked_paths) + # # No ack file found, warning message is posted + # self._test_result( + # self.record, + # {"edi_exchange_state": "output_sent_and_processed"}, + # state_paths=("done",), + # expected_messages=[ + # { + # "message": self.record._exchange_status_message("ack_missing"), + # "level": "warning", + # }, + # { + # "message": self.record._exchange_status_message("process_ok"), + # "level": "info", + # }, + # ], + # ) + + # @mute_logger(*LOGGERS) + # def test_export_file_already_done_ack_needed_found(self): + # self.record.edi_exchange_state = "output_sent" + # self.record.type_id.ack_needed = True + # mocked_paths = { + # self._file_fullpath("done"): self.fakepath, + # self._file_fullpath("done", ack=True): self.fakepath_ack, + # } + # self._test_run_cron(mocked_paths) + # # Found ack file, set on record + # self._test_result( + # self.record, + # { + # "edi_exchange_state": "output_sent_and_processed", + # "ack_file": base64.b64encode(b"ACK filecontent"), + # }, + # state_paths=("done",), + # expected_messages=[ + # { + # "message": self.record._exchange_status_message("ack_received"), + # "level": "info", + # }, + # { + # "message": self.record._exchange_status_message("process_ok"), + # "level": "info", + # }, + # ], + # ) + + @mute_logger(*LOGGERS) + def test_already_sent_process_error(self): + """Already sent, error process.""" + self.record.edi_exchange_state = "output_sent" + mocked_paths = { + self._file_fullpath("error"): self.fakepath, + self._file_fullpath("error-report"): self.fakepath_error, + } + self._test_run_cron(mocked_paths) + # As we simulate to find a file in `error` folder, + # we should get a call for: done, error and then the read of the report. + self._test_result( + self.record, + { + "edi_exchange_state": "output_sent_and_error", + "exchange_error": "ERROR XYZ: line 2 broken on bla bla", + }, + state_paths=("done", "error", "error-report"), + expected_messages=[ + { + "message": self.record._exchange_status_message("process_ko"), + "level": "error", + } + ], + ) + + @mute_logger(*LOGGERS) + def test_cron_full_flow(self): + """Already sent, update the state via cron.""" + self.record.edi_exchange_state = "output_sent" + rec1 = self.record + partner2 = self.env.ref("base.res_partner_2") + partner3 = self.env.ref("base.res_partner_3") + rec2 = self.record.copy( + { + "model": partner2._name, + "res_id": partner2.id, + "exchange_filename": "rec2.csv", + "exchange_file": rec1.exchange_file, + "edi_exchange_state": rec1.edi_exchange_state, + } + ) + rec3 = self.record.copy( + { + "model": partner3._name, + "res_id": partner3.id, + "exchange_filename": "rec3.csv", + "edi_exchange_state": "output_sent_and_error", + "exchange_file": rec1.exchange_file, + } + ) + # Avoid the exchange_file name generated from the compute function + # of rec2 and rec3 is the same, leading to an incorrect test + rec2.exchange_filename = "rec2.csv" + + mocked_paths = { + self._file_fullpath("done", record=rec1): self.fakepath, + self._file_fullpath("error", record=rec2): self.fakepath, + self._file_fullpath("error-report", record=rec2): self.fakepath_error, + self._file_fullpath("done", record=rec3): self.fakepath, + } + self._test_run_cron(mocked_paths) + self._test_result( + rec1, + {"edi_exchange_state": "output_sent_and_processed"}, + state_paths=("done",), + expected_messages=[ + { + "message": rec1._exchange_status_message("process_ok"), + "level": "info", + } + ], + ) + self._test_result( + rec2, + { + "edi_exchange_state": "output_sent_and_error", + "exchange_error": "ERROR XYZ: line 2 broken on bla bla", + }, + state_paths=("done", "error", "error-report"), + expected_messages=[ + { + "message": rec2._exchange_status_message("process_ko"), + "level": "error", + } + ], + ) + self._test_result( + rec3, + {"edi_exchange_state": "output_sent_and_processed"}, + state_paths=("done",), + expected_messages=[ + { + "message": rec3._exchange_status_message("process_ok"), + "level": "info", + } + ], + ) + + @mute_logger(*LOGGERS) + def test_create_input_exchange_file_from_file_received_no_pattern(self): + exch_type = self.exchange_type_in + exch_type.exchange_filename_pattern = "" + input_dir = "/test_input/pending/" + file_names = ["some-file.csv", "another-file.csv"] + self.backend.input_dir_pending = input_dir + mocked_paths = { + input_dir: "/tmp/", + self._file_fullpath( + "pending", fname=file_names[0], checker=self.checker_input + ): self.fakepath_input_pending_1, + self._file_fullpath( + "pending", fname=file_names[1], checker=self.checker_input + ): self.fakepath_input_pending_2, + } + existing_records = self.env["edi.exchange.record"].search( + [("backend_id", "=", self.backend.id), ("type_id", "=", exch_type.id)] + ) + # Run cron action: + found_files = [input_dir + fname for fname in file_names] + with self._mock_storage_backend_find_files(found_files): + self._test_run_cron_pending_input(mocked_paths) + new_records = self.env["edi.exchange.record"].search( + [ + ("backend_id", "=", self.backend.id), + ("type_id", "=", exch_type.id), + ("id", "not in", existing_records.ids), + ] + ) + self.assertEqual(len(new_records), 2) + for rec in new_records: + self.assertIn(rec.exchange_filename, file_names) + self.assertEqual(rec.edi_exchange_state, "input_pending") diff --git a/edi_storage_oca/tests/test_edi_storage_listener.py b/edi_storage_oca/tests/test_edi_storage_listener.py new file mode 100644 index 0000000000..aff811af19 --- /dev/null +++ b/edi_storage_oca/tests/test_edi_storage_listener.py @@ -0,0 +1,85 @@ +# Copyright 2021 ForgeFlow S.L. (https://www.forgeflow.com) +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +import base64 + +import mock + +from odoo.addons.edi_oca.tests.common import EDIBackendCommonComponentRegistryTestCase +from odoo.addons.edi_oca.tests.fake_components import FakeInputProcess + +LISTENER_MOCK_PATH = ( + "odoo.addons.edi_storage_oca.components.listener.EdiStorageListener" +) + + +class EDIBackendTestCase(EDIBackendCommonComponentRegistryTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls._load_module_components(cls, "edi_storage_oca") + cls._build_components( + cls, + FakeInputProcess, + ) + vals = { + "model": cls.partner._name, + "res_id": cls.partner.id, + "exchange_file": base64.b64encode(b"1234"), + } + cls.record = cls.backend.create_record("test_csv_input", vals) + cls.fake_move_args = None + + @classmethod + def _get_backend(cls): + return cls.env.ref("edi_storage_oca.demo_edi_backend_storage") + + def setUp(self): + super().setUp() + FakeInputProcess.reset_faked() + + def _move_file_mocked(self, *args): + self.fake_move_args = [*args] + if not all([*args]): + return False + return True + + def _mock_listener_move_file(self): + return mock.patch(LISTENER_MOCK_PATH + "._move_file", self._move_file_mocked) + + def _mock_listener_remove_file(self): + return mock.patch(LISTENER_MOCK_PATH + "._remove_file", self._move_file_mocked) + + def test_01_process_record_success(self): + with self._mock_listener_move_file(): + self.record.write({"edi_exchange_state": "input_received"}) + self.record.action_exchange_process() + storage, from_dir_str, to_dir_str, filename = self.fake_move_args + self.assertEqual(storage, self.backend.storage_id) + self.assertEqual(from_dir_str, self.backend.input_dir_pending) + self.assertEqual(to_dir_str, self.backend.input_dir_done) + self.assertEqual(filename, self.record.exchange_filename) + + def test_02_process_record_with_error(self): + with self._mock_listener_move_file(): + self.record.write({"edi_exchange_state": "input_received"}) + self.record._set_file_content("TEST %d" % self.record.id) + self.record.with_context( + test_break_process="OOPS! Something went wrong :(" + ).action_exchange_process() + storage, from_dir_str, to_dir_str, filename = self.fake_move_args + self.assertEqual(storage, self.backend.storage_id) + self.assertEqual(from_dir_str, self.backend.input_dir_pending) + self.assertEqual(to_dir_str, self.backend.input_dir_error) + self.assertEqual(filename, self.record.exchange_filename) + + def test_03_process_record_success_delete(self): + self.backend.write({"input_dir_remove": True, "input_dir_done": False}) + + with self._mock_listener_remove_file(): + self.record.write({"edi_exchange_state": "input_received"}) + self.record.action_exchange_process() + storage, from_dir_str, filename = self.fake_move_args + self.assertEqual(storage, self.backend.storage_id) + self.assertEqual(from_dir_str, self.backend.input_dir_pending) + self.assertEqual(filename, self.record.exchange_filename) diff --git a/edi_storage_oca/tests/test_exchange_type.py b/edi_storage_oca/tests/test_exchange_type.py new file mode 100644 index 0000000000..6b886eab3c --- /dev/null +++ b/edi_storage_oca/tests/test_exchange_type.py @@ -0,0 +1,67 @@ +# Copyright 2022 Camptocamp SA (https://www.camptocamp.com). +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +from odoo.addons.edi_oca.tests.common import EDIBackendCommonTestCase + + +class EDIExchangeTypeTestCase(EDIBackendCommonTestCase): + def _check_test_storage_fullpath(self, wanted_fullpath, directory, filename): + fullpath = self.exchange_type_out._storage_fullpath(directory, filename) + self.assertEqual(fullpath.as_posix(), wanted_fullpath) + + def _do_test_storage_fullpath(self, prefix=""): + # Test with no directory and no filename + wanted_fullpath = prefix or "." + self._check_test_storage_fullpath(wanted_fullpath, None, None) + + # Test with directory + directory = "test_directory" + wanted_fullpath = f"{prefix}/{directory}" if prefix else directory + self._check_test_storage_fullpath(wanted_fullpath, directory, None) + + # Test with filename + filename = "test_filename.csv" + wanted_fullpath = f"{prefix}/{filename}" if prefix else filename + self._check_test_storage_fullpath(wanted_fullpath, None, filename) + + # Test with directory and filename + wanted_fullpath = ( + f"{prefix}/{directory}/{filename}" if prefix else f"{directory}/{filename}" + ) + self._check_test_storage_fullpath(wanted_fullpath, directory, filename) + + def test_storage_fullpath(self): + """ + Test storage fullpath defined into advanced settings. + Example of pattern: + storage: + # simple string + path: path/to/file + # name of the param containing the path + path_config_param: path/to/file + """ + + # Test without any prefix + self._do_test_storage_fullpath() + + # Force path on advanced settings + prefix = "prefix/path" + self.exchange_type_out.advanced_settings_edit = f""" + storage: + path: {prefix} + """ + self._do_test_storage_fullpath(prefix=prefix) + + # Force path on advanced settings using config param, but not defined + self.exchange_type_out.advanced_settings_edit = """ + storage: + path_config_param: prefix_path_config_param + """ + self._do_test_storage_fullpath() + + # Define config param + prefix = "prefix/path/by/config/param" + self.env["ir.config_parameter"].sudo().set_param( + "prefix_path_config_param", prefix + ) + self._do_test_storage_fullpath(prefix=prefix) diff --git a/edi_storage_oca/views/edi_backend_views.xml b/edi_storage_oca/views/edi_backend_views.xml new file mode 100644 index 0000000000..59ed64b0c7 --- /dev/null +++ b/edi_storage_oca/views/edi_backend_views.xml @@ -0,0 +1,26 @@ + + + + edi.backend + + + + + + + + + + + + + + + + + + + diff --git a/setup/edi_storage_oca/odoo/addons/edi_storage_oca b/setup/edi_storage_oca/odoo/addons/edi_storage_oca new file mode 120000 index 0000000000..7c6d68f63f --- /dev/null +++ b/setup/edi_storage_oca/odoo/addons/edi_storage_oca @@ -0,0 +1 @@ +../../../../edi_storage_oca \ No newline at end of file diff --git a/setup/edi_storage_oca/setup.py b/setup/edi_storage_oca/setup.py new file mode 100644 index 0000000000..28c57bb640 --- /dev/null +++ b/setup/edi_storage_oca/setup.py @@ -0,0 +1,6 @@ +import setuptools + +setuptools.setup( + setup_requires=['setuptools-odoo'], + odoo_addon=True, +) diff --git a/test-requirements.txt b/test-requirements.txt index 5c4163411a..7da47772d9 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,3 +1,5 @@ invoice2data==0.3.5 freezegun odoo_test_helper + +odoo12-addon-storage_backend @ git+https://github.com/OCA/storage.git@refs/pull/367/head#subdirectory=setup/storage_backend