feat(retropath2): add Preference object and add it to cmd line for knime

.github/workflows/test.yml

@@ -19,7 +19,7 @@ jobs:
     runs-on: ${{ matrix.os }}-latest
     strategy:
       matrix:
-        os: ["windows"]
+        os: ["macos-latest", "ubuntu-latest", "windows-latest"]
     defaults:
       run:
         shell: bash -l {0}

retropath2_wrapper/Args.py

@@ -9,7 +9,7 @@
 from retropath2_wrapper._version import __version__
 
 
-DEFAULT_TIMEOUT = 60  # minutes
+DEFAULT_TIMEOUT = 5  # minutes
 DEFAULT_KNIME_VERSION = '4.5.0'
 DEFAULT_RP2_VERSION = 'r20220104'
 KNIME_PACKAGE = {

retropath2_wrapper/RetroPath2.py

@@ -61,6 +61,8 @@
     KNIME_PACKAGE,
     RETCODES,
 )
+from retropath2_wrapper.preference import Preference
+
 
 here = os_path.dirname(os_path.realpath(__file__))
 
@@ -121,8 +123,6 @@ def set_vars(
         here, 'workflows', f'RetroPath2.0_{rp2_version}.knwf'
     )
 
-    if sys_platform == 'win32':
-        workflow = "/".join(workflow.split(os.sep))
 
     # Build a dict to store KNIME vars
     return {
@@ -236,6 +236,9 @@ def retropath2(
 
     logger.info('{attr1}Initializing{attr2}'.format(attr1=attr('bold'), attr2=attr('reset')))
 
+    # Preferences
+    preference = Preference(rdkit_timeout_minutes=timeout)
+
     with TemporaryDirectory() as tempd:
 
         # Format files for KNIME
@@ -252,11 +255,11 @@ def retropath2(
 
         # Call KNIME
         r_code = call_knime(
-            kvars,
-            files,
-            rp2_params,
-            timeout,
-            logger
+            kvars=kvars,
+            files=files,
+            params=rp2_params,
+            preference=preference,
+            logger=logger,
         )
         if r_code == RETCODES['TimeLimit'] or r_code == RETCODES['OSError']:
             return r_code, files
@@ -496,6 +499,11 @@ def gunzip_to_csv(filename: str, indir: str) -> str:
     return filename
 
 
+def standardize_path(path: str) -> str:
+    if sys_platform == 'win32':
+        path = "/".join(path.split(os.sep))
+    return path
+
 def format_files_for_knime(
     sinkfile: str, sourcefile: str, rulesfile: str,
     indir: str, outdir: str,
@@ -551,9 +559,6 @@ def format_files_for_knime(
             copyfile(files[key], new_f)
             files[key] = new_f
 
-    if sys_platform == 'win32':
-        for key, value in files.items():
-            files[key] = "/".join(value.split(os.sep))
     return files
 
 
@@ -605,7 +610,7 @@ def call_knime(
     kvars: Dict,
     files: Dict,
     params: Dict,
-    timeout: int,
+    preference: Preference,
     logger: Logger = getLogger(__name__)
 ) -> int:
     """
@@ -619,8 +624,8 @@ def call_knime(
         Paths of sink, source, rules files.
     params: Dict
         Parameters of the workflow to process.
-    timeout: int
-        Time after which the run returns.
+    preference: Preference
+        A preference object.
     logger : Logger
         The logger object.
 
@@ -633,25 +638,32 @@ def call_knime(
     StreamHandler.terminator = ""
     logger.info('{attr1}Running KNIME...{attr2}'.format(attr1=attr('bold'), attr2=attr('reset')))
 
-    preference = os_path.join(
-        here, 'workflows', 'preferences.epf'
-    )
-    args = ' -nosplash -nosave -reset -consoleLog --launcher.suppressErrors -application org.knime.product.KNIME_BATCH_APPLICATION ' \
-         + ' -workflowFile=' + kvars['workflow'] \
-         + ' -workflow.variable=input.dmin,"'              + str(params['dmin'])         + '",int' \
-         + ' -workflow.variable=input.dmax,"'              + str(params['dmax'])         + '",int' \
-         + ' -workflow.variable=input.max-steps,"'         + str(params['max_steps'])    + '",int' \
-         + ' -workflow.variable=input.sourcefile,"'        + files['source']             + '",String' \
-         + ' -workflow.variable=input.sinkfile,"'          + files['sink']               + '",String' \
-         + ' -workflow.variable=input.rulesfile,"'         + files['rules']              + '",String' \
-         + ' -workflow.variable=input.topx,"'              + str(params['topx'])         + '",int' \
-         + ' -workflow.variable=input.mwmax-source,"'      + str(params['mwmax_source']) + '",int' \
-         + ' -workflow.variable=output.dir,"'              + files['outdir']             + '",String' \
-         + ' -workflow.variable=output.solutionfile,"'     + files['results']            + '",String' \
-         + ' -workflow.variable=output.sourceinsinkfile,"' + files['src-in-sk']          + '",String' \
-         + ' -preferences=' + preference
-
-    logger.debug(kvars['kexec'] + ' ' + args)
+    args = [kvars["kexec"]]
+    args += ["-nosplash"]
+    args += ["-nosave"]
+    args += ["-reset"]
+    args += ["-consoleLog"]
+    args += ["--launcher.suppressErrors"]
+    args += ["-application", "org.knime.product.KNIME_BATCH_APPLICATION"]
+    args += ["-workflowFile=%s" % (standardize_path(path=kvars['workflow']),)]
+
+    args += ['-workflow.variable=input.dmin,"%s",int' % (params['dmin'],)]
+    args += ['-workflow.variable=input.dmax,"%s",int' % (params['dmax'],)]
+    args += ['-workflow.variable=input.max-steps,"%s",int' % (params['max_steps'],)]
+    args += ['-workflow.variable=input.topx,"%s",int' % (params['topx'],)]
+    args += ['-workflow.variable=input.mwmax-source,"%s",int' % (params['mwmax_source'],)]
+
+    args += ['-workflow.variable=input.sourcefile,"%s",String' % (standardize_path(files['source']),)]
+    args += ['-workflow.variable=input.sinkfile,"%s",String' % (standardize_path(files['sink']),)]
+    args += ['-workflow.variable=input.rulesfile,"%s",String' % (standardize_path(files['rules']),)]
+    args += ['-workflow.variable=output.dir,"%s",String' % (standardize_path(files['outdir']),)]
+    args += ['-workflow.variable=output.solutionfile,"%s",String' % (standardize_path(files['results']),)]
+    args += ['-workflow.variable=output.sourceinsinkfile,"%s",String' % (standardize_path(files['src-in-sk']),)]
+    if preference and preference.is_init():
+        preference.to_file()
+        args += ["-preferences=" + standardize_path(preference.path)]
+
+    logger.debug(" ".join(args))
 
     try:
         printout = open(devnull, 'wb') if logger.level > 10 else None
@@ -663,7 +675,7 @@ def call_knime(
             os_environ['CONDA_PREFIX'],
             "lib"
         )
-        returncode = subprocess_call(cmd=kvars['kexec'] + args, logger=logger)
+        returncode = subprocess_call(cmd=" ".join(args), logger=logger)
         os_environ['LD_LIBRARY_PATH'] = ':'.join(
             os_environ['LD_LIBRARY_PATH'].split(':')[:-1]
         )
@@ -672,7 +684,7 @@ def call_knime(
         return returncode
 
     except TimeoutExpired as e:
-        logger.warning('   |- Time limit ({timeout} min) is reached'.format(timeout=timeout))
+        logger.warning('   |- Time limit ({timeout} min) is reached'.format(timeout=preference.rdkit_timeout_minutes))
         logger.warning('      Results collected until now are available')
         return RETCODES['TimeLimit']
 

retropath2_wrapper/preference.py

@@ -0,0 +1,26 @@
+import datetime
+import tempfile
+
+
+class Preference(object):
+    def __init__(self, *args, **kwargs) -> None:
+        self.path = kwargs.get("path", tempfile.NamedTemporaryFile(suffix=".epf").name)
+        self.rdkit_timeout_minutes = kwargs.get("rdkit_timeout_minutes")
+
+    def is_init(self) -> bool:
+        if self.rdkit_timeout_minutes:
+            return True
+        return False
+
+    def to_file(self) -> None:
+        now = datetime.datetime.now(datetime.timezone.utc)
+        with open(self.path, "w") as fod:
+            fod.write("#")
+            fod.write(now.strftime("%a %b %d %H:%M:%S %Z %Y"))
+            fod.write("\n")
+            fod.write("\\!/=")
+            fod.write("\n")
+            if self.rdkit_timeout_minutes:
+                fod.write("/instance/org.rdkit.knime.nodes/mcsAggregation.timeout=")
+                fod.write(str(int(self.rdkit_timeout_minutes) * 60))
+                fod.write("\n")

tests/data/preference.epf

@@ -0,0 +1,3 @@
+#Thu Nov 17 14:02:50 CET 2022
+\!/=
+/instance/org.rdkit.knime.nodes/mcsAggregation.timeout=600

tests/functional/test_retropath2.py

@@ -6,6 +6,7 @@
 import filecmp
 import os
 import shutil
+import sys
 import tempfile
 
 from retropath2_wrapper.__main__ import create_logger
@@ -18,7 +19,6 @@ class TestRetropath2(Main_test):
     def setUp(self):
         self.logger = create_logger(__name__, 'DEBUG')
 
-    '''
     def test_src_in_sink(self):
         tmpdir = tempfile.mkdtemp()
         r_code, result = retropath2(
@@ -30,25 +30,31 @@ def test_src_in_sink(self):
         )
         self.assertEqual(r_code, RETCODES['SrcInSink'])
         shutil.rmtree(tmpdir, ignore_errors=True)
-    '''
+
     def test_lycopene(self):
         tmpdir = tempfile.mkdtemp()
         r_code, result = retropath2(
             sink_file=self.lycopene_sink_csv,
             source_file=self.lycopene_source_csv,
             rules_file=self.rulesd12_csv,
             outdir=tmpdir,
+            timeout=10,
             logger=self.logger,
         )
-        # Compare number of lines to have a robust test accross platforms
-        with open(result['outdir'] + "/" + result['results']) as fid:
-            result_lines = fid.read().splitlines()
-        with open(self.lycopene_r20220104_results_csv) as fid:
-            theorical_lines = fid.read().splitlines()
+        if sys.platform == 'win32':
+            with open(result['outdir'] + "/" + result['results']) as fid:
+                result_lines = fid.read().splitlines()
+            with open(self.lycopene_r20220104_results_csv) as fid:
+                theorical_lines = fid.read().splitlines()
 
-        shutil.copyfile(result['outdir'] + "/" + result['results'], os.path.join(self.dataset_path, "results_windows.csv"))
-        self.assertTrue(False == True)
-        # self.assertEqual(len(result_lines), len(theorical_lines))
+            identical_line = 0
+            for i, the in enumerate(theorical_lines):
+                if the != result_lines[i]:
+                    identical_line = i
+                    break
+            self.assertTrue(identical_line > 5)
+        else:
+            filecmp.cmp(result['outdir'] + "/" + result['results'], self.lycopene_r20220104_results_csv)
         shutil.rmtree(tmpdir, ignore_errors=True)
 
     """

tests/main_test.py

@@ -5,6 +5,9 @@
 class Main_test(unittest.TestCase):
     dataset_path = os.path.join(os.path.dirname(__file__), "data")
 
+    # preferences
+    preference = os.path.join(dataset_path, "preference.epf")
+
     # rules
     rules_csv = os.path.join(dataset_path, "rules.csv.gz")
     rulesd12_csv = os.path.join(dataset_path, "rules_d12.csv.gz")

tests/test_helpers.py

@@ -7,7 +7,7 @@
 import tempfile
 
 from retropath2_wrapper.Args import RETCODES
-from retropath2_wrapper.RetroPath2 import check_inchi_from_file, check_input
+from retropath2_wrapper.RetroPath2 import check_inchi_from_file, check_input, standardize_path
 from tests.main_test import Main_test
 
 
@@ -38,3 +38,9 @@ def test_check_inchi_from_file(self):
             fod.close()
             self.assertNotEqual(check_inchi_from_file(fod.name), "")
             os.remove(fod.name)
+
+    def test_standardize_path(self):
+        path = os.getcwd()
+
+        spath = standardize_path(path=path)
+        self.assertTrue("\\" not in spath)

tests/test_preference.py

@@ -0,0 +1,35 @@
+"""
+Created on Jul 15 2020
+
+@author: Joan Hérisson
+"""
+import os
+import tempfile
+
+from retropath2_wrapper.preference import Preference
+from tests.main_test import Main_test
+
+
+class TestPreference(Main_test):
+    def test_init(self):
+        pref = Preference(path=os.getcwd(), rdkit_timeout_minutes=50, rdkit="test")
+        self.assertEqual(pref.path, os.getcwd())
+        self.assertEqual(pref.rdkit_timeout_minutes, 50)
+        with self.assertRaises(Exception):
+            pref.rdkit
+
+    def test_to_file(self):
+        pref = Preference(rdkit_timeout_minutes=10)
+        pref.to_file()
+        with open(pref.path) as fid:
+            res = fid.read().splitlines()
+        with open(self.preference) as fid:
+            the = fid.read().splitlines()
+        self.assertEqual(res[1:], the[1:])
+
+    def test_is_init(self):
+        pref = Preference()
+        self.assertFalse(pref.is_init())
+        pref = Preference(rdkit_timeout_minutes=10)
+        self.assertTrue(pref.is_init())
+