Add all expid_dir_path functions to autosubmit.py and fix tests !509

All previous definitions all expid_path, tmp_path, log_path and
aslogs_path (and similar) have been removed and their calls have been
switched for BasicConfig.foo() calls. Impacted tests are mainly affected
in terms of adding Mock objects since BasicConfig.foo() functions are
not available in autosubmitconfigparser yet. No new tests added

autosubmit/notifications/mail_notifier.py

@@ -20,18 +20,30 @@
 import smtplib
 import email.utils
 from email.mime.text import MIMEText
+from email.mime.multipart import MIMEMultipart
+from email.mime.application import MIMEApplication
 from log.log import Log
+from pathlib import Path
+from autosubmitconfigparser.config.basicconfig import BasicConfig
 
 class MailNotifier:
     def __init__(self, basic_config):
         self.config = basic_config
 
     def notify_experiment_status(self, exp_id,mail_to,platform):
         message_text = self._generate_message_experiment_status(exp_id, platform)
-        message = MIMEText(message_text)
+        message = MIMEMultipart() 
         message['From'] = email.utils.formataddr(('Autosubmit', self.config.MAIL_FROM))
-        message['Subject'] = f'[Autosubmit] Warning a remote platform is malfunctioning'
+        message['Subject'] = f'[Autosubmit] Warning: a remote platform is malfunctioning'
         message['Date'] = email.utils.formatdate(localtime=True)
+        message.attach(MIMEText(message_text))
+
+        try:
+            files = [f for f in BasicConfig.expid_aslog_dir(exp_id).glob('*_run.log') if Path(f).is_file()]
+            self._attach_files(message, files)
+        except BaseException as e:
+            Log.printlog('An error has occurred while attaching log files to a warning email about remote_platforms ', 6011)
+
         for mail in mail_to:
             message['To'] = email.utils.formataddr((mail, mail))
             try:
@@ -53,26 +65,34 @@ def notify_status_change(self, exp_id, job_name, prev_status, status, mail_to):
                 Log.printlog('Trace:{0}\nAn error has occurred while sending a mail for the job {0}'.format(e,job_name), 6011)
 
     def _send_mail(self, mail_from, mail_to, message):
-        server = smtplib.SMTP(self.config.SMTP_SERVER,timeout=60)
+        server = smtplib.SMTP(self.config.SMTP_SERVER, timeout=60)
         server.sendmail(mail_from, mail_to, message.as_string())
         server.quit()
 
+    def _attach_files(self, message, files):
+        for f in files or []:
+            with open(f, "rb") as file:
+                part = MIMEApplication(file.read(), Name = Path(f).name)
+                part['Content-Disposition'] = 'attachment; filename="%s"' % Path(f).name
+                message.attach(part)
+
+
     @staticmethod
     def _generate_message_text(exp_id, job_name, prev_status, status):
-        return f'Autosubmit notification\n' \
-               f'-------------------------\n\n' \
-                f'Experiment id:  {str(exp_id)} \n\n' \
-                + f'Job name: {str(job_name)} \n\n' \
-                f'The job status has changed from: {str(prev_status)} to {str(status)} \n\n\n\n\n' \
-                f'INFO: This message was auto generated by Autosubmit, '\
-                f'remember that you can disable these messages on Autosubmit config file. \n'
+        return f'''Autosubmit notification\n
+               -------------------------\n\n
+               Experiment id:  {str(exp_id)} \n\n
+               Job name: {str(job_name)} \n\n
+               The job status has changed from: {str(prev_status)} to {str(status)} \n\n\n\n\n
+               INFO: This message was auto generated by Autosubmit,
+               remember that you can disable these messages on Autosubmit config file. \n'''
 
     @staticmethod
     def _generate_message_experiment_status(exp_id, platform=""):
-        return f'Autosubmit notification: Remote Platform issues\n' \
-               f'-------------------------\n\n' \
-                f'Experiment id:{str(exp_id)} \n\n' \
-                + f'Platform affected:{str(platform.name)} using as host:{str(platform.host)} \n\n' \
-                f'[WARN] Autosubmit encountered an issue with an remote_platform.\n It will resume itself, whenever is possible\n If issue persist, you can change the host IP or put multiple hosts in the platform.yml' + '\n\n\n\n\n' \
-                f'INFO: This message was auto generated by Autosubmit, '\
-                f'remember that you can disable these messages on Autosubmit config file. \n'
+        return f'''Autosubmit notification: Remote Platform issues\n-------------------------\n
+               Experiment id: {str(exp_id)}
+               Logs and errors: {BasicConfig.expid_aslog_dir(exp_id)}
+               Attached to this message you will find the related _run.log files.
+
+               Platform affected: {str(platform.name)} using as host: {str(platform.host)}\n\n[WARN] Autosubmit encountered an issue with a remote platform.\n It will resume itself, whenever is possible\n If this issue persists, you can change the host IP or put multiple hosts in the platform.yml file' + '\n\n\n\n\nINFO: This message was auto generated by Autosubmit,
+                remember that you can disable these messages on Autosubmit config file.\n'''

test/unit/conftest.py

@@ -8,6 +8,7 @@
 from shutil import rmtree
 from tempfile import TemporaryDirectory
 from typing import Any, Dict, Callable, List, Protocol, Optional
+import os
 
 from autosubmit.autosubmit import Autosubmit
 from autosubmit.platforms.slurmplatform import SlurmPlatform, ParamikoPlatform
@@ -27,7 +28,6 @@ class AutosubmitExperiment:
     status_dir: Path
     platform: ParamikoPlatform
 
-
 @pytest.fixture(scope='function')
 def autosubmit_exp(autosubmit: Autosubmit, request: pytest.FixtureRequest) -> Callable:
     """Create an instance of ``Autosubmit`` with an experiment."""
@@ -36,17 +36,21 @@ def autosubmit_exp(autosubmit: Autosubmit, request: pytest.FixtureRequest) -> Ca
     tmp_dir = TemporaryDirectory()
     tmp_path = Path(tmp_dir.name)
 
+
     def _create_autosubmit_exp(expid: str):
-        # directories used when searching for logs to cat
         root_dir = tmp_path
         BasicConfig.LOCAL_ROOT_DIR = str(root_dir)
-        exp_path = root_dir / expid
-        exp_tmp_dir = exp_path / BasicConfig.LOCAL_TMP_DIR
-        aslogs_dir = exp_tmp_dir / BasicConfig.LOCAL_ASLOG_DIR
-        status_dir = exp_path / 'status'
-        aslogs_dir.mkdir(parents=True, exist_ok=True)
-        status_dir.mkdir(parents=True, exist_ok=True)
-
+        exp_path = BasicConfig.expid_dir(expid)
+
+        # directories used when searching for logs to cat
+        exp_tmp_dir = BasicConfig.expid_tmp_dir(expid) 
+        aslogs_dir = BasicConfig.expid_aslog_dir(expid) 
+        status_dir =exp_path / 'status'
+        if not os.path.exists(aslogs_dir):
+            os.makedirs(aslogs_dir)
+        if not os.path.exists(status_dir):
+            os.makedirs(status_dir)
+
         platform_config = {
             "LOCAL_ROOT_DIR": BasicConfig.LOCAL_ROOT_DIR,
             "LOCAL_TMP_DIR": str(exp_tmp_dir),
@@ -59,7 +63,7 @@ def _create_autosubmit_exp(expid: str):
             'QUEUING': [],
             'FAILED': []
         }
-        submit_platform_script = aslogs_dir / 'submit_local.sh'
+        submit_platform_script = aslogs_dir.joinpath('submit_local.sh')
         submit_platform_script.touch(exist_ok=True)
 
         return AutosubmitExperiment(
@@ -94,7 +98,7 @@ def autosubmit() -> Autosubmit:
 @pytest.fixture(scope='function')
 def create_as_conf() -> Callable:  # May need to be changed to use the autosubmit_config one
     def _create_as_conf(autosubmit_exp: AutosubmitExperiment, yaml_files: List[Path], experiment_data: Dict[str, Any]):
-        conf_dir = autosubmit_exp.exp_path / 'conf'
+        conf_dir = autosubmit_exp.exp_path.joinpath('conf')
         conf_dir.mkdir(parents=False, exist_ok=False)
         basic_config = BasicConfig
         parser_factory = YAMLParserFactory()
@@ -117,7 +121,6 @@ def _create_as_conf(autosubmit_exp: AutosubmitExperiment, yaml_files: List[Path]
 
     return _create_as_conf
 
-
 class AutosubmitConfigFactory(Protocol):  # Copied from the autosubmit config parser, that I believe is a revised one from the create_as_conf
 
     def __call__(self, expid: str, experiment_data: Optional[Dict], *args: Any, **kwargs: Any) -> AutosubmitConfig: ...

test/unit/test_catlog.py

@@ -6,25 +6,33 @@
 from pathlib import Path
 from tempfile import TemporaryDirectory
 from unittest.mock import patch
+import pytest
 
 from autosubmit.autosubmit import Autosubmit, AutosubmitCritical
 from autosubmitconfigparser.config.basicconfig import BasicConfig
 
-
 class TestJob(TestCase):
 
     def setUp(self):
-        self.autosubmit = Autosubmit()
-        # directories used when searching for logs to cat
         self.original_root_dir = BasicConfig.LOCAL_ROOT_DIR
         self.root_dir = TemporaryDirectory()
         BasicConfig.LOCAL_ROOT_DIR = self.root_dir.name
-        self.exp_path = Path(self.root_dir.name, 'a000')
-        self.tmp_dir = self.exp_path / BasicConfig.LOCAL_TMP_DIR
-        self.aslogs_dir = self.tmp_dir / BasicConfig.LOCAL_ASLOG_DIR
-        self.status_path = self.exp_path / 'status'
-        self.aslogs_dir.mkdir(parents=True)
-        self.status_path.mkdir()
+
+        self.exp_path = BasicConfig.expid_dir('a000') 
+        self.tmp_dir = BasicConfig.expid_tmp_dir('a000')
+        self.log_dir = BasicConfig.expid_log_dir('a000')
+        self.aslogs_dir = BasicConfig.expid_aslog_dir('a000') 
+
+        self.autosubmit = Autosubmit()
+        # directories used when searching for logs to cat
+
+        self.status_path = self.exp_path / 'status' 
+        if not self.aslogs_dir.exists():
+            self.aslogs_dir.mkdir(parents = True, exist_ok = True)
+        if not self.status_path.exists():
+            self.status_path.mkdir(parents = True, exist_ok = True)
+        if not self.log_dir.exists():
+            self.log_dir.mkdir(parents=True, exist_ok=True)
 
     def tearDown(self) -> None:
         BasicConfig.LOCAL_ROOT_DIR = self.original_root_dir
@@ -55,15 +63,17 @@ def test_is_workflow_not_found(self, Log):
         assert Log.info.call_args[0][0] == 'No logs found.'
 
     def test_is_workflow_log_is_dir(self):
-        log_file_actually_dir = Path(self.aslogs_dir, 'log_run.log')
-        log_file_actually_dir.mkdir()
+        log_file_actually_dir = self.aslogs_dir / 'log_run.log'
+        log_file_actually_dir.mkdir(parents=True)
         def _fn():
             self.autosubmit.cat_log('a000', 'o', 'c')
         self.assertRaises(AutosubmitCritical, _fn)
 
     @patch('subprocess.Popen')
     def test_is_workflow_out_cat(self, popen):
-        log_file = Path(self.aslogs_dir, 'log_run.log')
+        log_file = self.aslogs_dir / 'log_run.log'
+        if log_file.is_dir(): # dir is created in previous test 
+            log_file.rmdir()
         with open(log_file, 'w') as f:
             f.write('as test')
             f.flush()
@@ -75,7 +85,7 @@ def test_is_workflow_out_cat(self, popen):
 
     @patch('subprocess.Popen')
     def test_is_workflow_status_tail(self, popen):
-        log_file = Path(self.status_path, 'a000_anything.txt')
+        log_file = self.status_path / 'a000_anything.txt'
         with open(log_file, 'w') as f:
             f.write('as test')
             f.flush()
@@ -93,19 +103,19 @@ def test_is_jobs_not_found(self, Log):
             self.autosubmit.cat_log('a000_INI', file=file, mode='c')
             assert Log.info.called
             assert Log.info.call_args[0][0] == 'No logs found.'
-
+    
     def test_is_jobs_log_is_dir(self):
-        log_file_actually_dir = Path(self.tmp_dir, 'LOG_a000/a000_INI.20000101.out')
+        log_file_actually_dir = self.log_dir / 'a000_INI.20000101.out'
         log_file_actually_dir.mkdir(parents=True)
         def _fn():
             self.autosubmit.cat_log('a000_INI', 'o', 'c')
         self.assertRaises(AutosubmitCritical, _fn)
 
     @patch('subprocess.Popen')
     def test_is_jobs_out_tail(self, popen):
-        log_dir = self.tmp_dir / 'LOG_a000'
-        log_dir.mkdir()
-        log_file = log_dir / 'a000_INI.20200101.out'
+        log_file = self.log_dir / 'a000_INI.20200101.out'
+        if log_file.is_dir(): # dir is created in previous test 
+            log_file.rmdir()
         with open(log_file, 'w') as f:
             f.write('as test')
             f.flush()

test/unit/test_describe.py

@@ -33,29 +33,35 @@ def test_describe(
         if expid not in _EXPIDS:
             continue
         exp = autosubmit_exp(expid)
-
+    
         basic_config = mocker.MagicMock()
+        # TODO: Whenever autosubmitconfigparser gets released with BasicConfig.expid_dir() and similar functions, the following line and a lot of mocks need to be removed. This line is especially delicate because it "overmocks" the basic_config mock, thus making the last assertion of this file "assert f'Location: {exp.exp_path}' in log_result_output" a dummy assertion. The rest of the test is still useful.
+        basic_config.expid_dir.side_effect = lambda expid: exp.exp_path
         config_values = {
             'LOCAL_ROOT_DIR': str(exp.exp_path.parent),
             'LOCAL_ASLOG_DIR': str(exp.aslogs_dir)
         }
+
         for key, value in config_values.items():
             basic_config.__setattr__(key, value)
+
         basic_config.get.side_effect = lambda key, default='': config_values.get(key, default)
         for basic_config_location in [
             'autosubmit.autosubmit.BasicConfig',
             'autosubmitconfigparser.config.configcommon.BasicConfig'
         ]:
             # TODO: Better design how ``BasicConfig`` is used/injected/IOC/etc..
             mocker.patch(basic_config_location, basic_config)
+
         mocked_get_submitter = mocker.patch.object(Autosubmit, '_get_submitter')
         submitter = mocker.MagicMock()
+
         mocked_get_submitter.return_value = submitter
         submitter.platforms = [1, 2]
 
         get_experiment_descrip = mocker.patch('autosubmit.autosubmit.get_experiment_descrip')
         get_experiment_descrip.return_value = [[f'{expid} description']]
-
+        
         create_as_conf(
             autosubmit_exp=exp,
             yaml_files=[
@@ -69,6 +75,7 @@ def test_describe(
             }
         )
 
+
     Autosubmit.describe(
         input_experiment_list=input_experiment_list,
         get_from_user=get_from_user
@@ -84,4 +91,4 @@ def test_describe(
         ]
         root_dir = exp.exp_path.parent
         for expid in expids:
-            assert f'Location: {str(root_dir / expid)}' in log_result_output
+            assert f'Location: {exp.exp_path}' in log_result_output