Allow use import from the notebook

ipython2cwl/repo2cwl.py

@@ -23,7 +23,8 @@
 
 def _get_notebook_paths_from_dir(dir_path: str):
     notebooks_paths = []
-    for path, _, files in os.walk(dir_path):
+    for path, dirs, files in os.walk(dir_path):
+        dirs[:] = [d for d in dirs if d not in "cwl"]
         for name in files:
             if name.endswith('.ipynb'):
                 notebooks_paths.append(os.path.join(path, name))
@@ -40,16 +41,14 @@ def _store_jn_as_script(notebook_path: str, git_directory_absolute_path: str, bi
     if len(converter._variables) == 0:
         logger.info(f"Notebook {notebook_path} does not contains typing annotations. skipping...")
         return None, None
-    script_relative_path = os.path.relpath(notebook_path, git_directory_absolute_path)[:-6]
-    script_relative_parent_directories = script_relative_path.split(os.sep)
-    if len(script_relative_parent_directories) > 1:
-        script_absolute_name = os.path.join(bin_absolute_path, os.sep.join(script_relative_parent_directories[:-1]))
-        os.makedirs(
-            script_absolute_name,
-            exist_ok=True)
-        script_absolute_name = os.path.join(script_absolute_name, os.path.basename(script_relative_path))
-    else:
-        script_absolute_name = os.path.join(bin_absolute_path, script_relative_path)
+
+    #change the extension from ipynb to nothing
+    notebook_absolute = Path(notebook_path)
+    notebook_relative_to_git = notebook_absolute.relative_to(git_directory_absolute_path)
+    notebook_name_without_extension = notebook_absolute.stem
+    script_absolute = Path(bin_absolute_path) / notebook_relative_to_git.parent / notebook_name_without_extension
+    #write the script, return it relative to
+    script_relative_to_git = script_absolute.relative_to(bin_absolute_path)
     script = os.linesep.join([
         '#!/usr/bin/env ipython',
         '"""',
@@ -59,13 +58,12 @@ def _store_jn_as_script(notebook_path: str, git_directory_absolute_path: str, bi
         '"""\n\n',
         converter._wrap_script_to_method(converter._tree, converter._variables)
     ])
-    with open(script_absolute_name, 'w') as fd:
+    with open(script_absolute, 'w') as fd:
         fd.write(script)
     tool = converter.cwl_command_line_tool(image_id)
-    in_git_dir_script_file = os.path.join(bin_absolute_path, script_relative_path)
-    tool_st = os.stat(in_git_dir_script_file)
-    os.chmod(in_git_dir_script_file, tool_st.st_mode | stat.S_IEXEC)
-    return tool, script_relative_path
+    tool_st = os.stat(script_absolute)
+    os.chmod(script_absolute, tool_st.st_mode | stat.S_IEXEC)
+    return tool, str(script_relative_to_git)
 
 
 def existing_path(path_str: str):
@@ -126,7 +124,8 @@ def repo2cwl(argv: Optional[List[str]] = None) -> int:
     image_id, cwl_tools = _repo2cwl(local_git)
     logger.info(f'Generated image id: {image_id}')
     for tool in cwl_tools:
-        base_command_script_name = f'{tool["baseCommand"][len("/app/cwl/bin/"):].replace("/", "_")}.cwl'
+        script_name_path = Path(tool["baseCommand"]).stem
+        base_command_script_name = f"""{str(script_name_path)}.cwl"""
         tool_filename = str(output_directory.joinpath(base_command_script_name))
         with open(tool_filename, 'w') as f:
             logger.info(f'Creating CWL command line tool: {tool_filename}')
@@ -148,9 +147,11 @@ def _repo2cwl(git_directory_path: Repo) -> Tuple[str, List[Dict]]:
     r2d.target_repo_dir = os.path.join(os.path.sep, 'app')
     r2d.repo = git_directory_path.tree().abspath
     bin_path = os.path.join(r2d.repo, 'cwl', 'bin')
-    os.makedirs(bin_path, exist_ok=True)
-    notebooks_paths = _get_notebook_paths_from_dir(r2d.repo)
+    shutil.copytree(r2d.repo, bin_path)
 
+    notebooks_paths = _get_notebook_paths_from_dir(r2d.repo)
+    logger.info(notebooks_paths)
+
     tools = []
     for notebook in notebooks_paths:
         cwl_command_line_tool, script_name = _store_jn_as_script(

tests/repo-like-with-imports/example1_import.ipynb

@@ -0,0 +1,75 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from ipython2cwl.iotypes import CWLFilePathInput, CWLStringInput, CWLFilePathOutput\n",
+    "from typing import List\n",
+    "import yaml\n",
+    "import os\n",
+    "\n",
+    "import simple_import"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "datafilename: CWLFilePathInput = 'data.yaml'\n",
+    "messages: List[CWLStringInput] = ['hello', 'world']\n",
+    "with open(datafilename) as fd: \n",
+    "    data = yaml.safe_load(fd)\n",
+    "print(data)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data['entry1'] += 1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "results_filename: CWLFilePathOutput = 'result.yaml'\n",
+    "with open(results_filename, 'w') as fd:\n",
+    "    yaml.safe_dump(data, fd)\n",
+    "messages_outputs: CWLFilePathOutput = 'messages.txt'\n",
+    "with open(messages_outputs, 'w') as f:\n",
+    "    f.write(' '.join(messages))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

tests/repo-like-with-imports/requirements.txt

@@ -0,0 +1 @@
+PyYAML==5.3.1

tests/repo-like-with-imports/simple_import.py

@@ -0,0 +1,2 @@
+def hi():
+    print("hi")

tests/simple_import.ipynb

@@ -0,0 +1,97 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "from typing import List, Optional\n",
+    "import matplotlib\n",
+    "from ipython2cwl.iotypes import CWLFilePathInput, CWLFilePathOutput, CWLStringInput"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dataset: CWLFilePathInput = './data/data.csv'\n",
+    "messages: List[CWLStringInput] = ['hello', 'world']\n",
+    "optional_message: Optional[CWLStringInput] = \"Hello from optional\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data = pd.read_csv(dataset)\n",
+    "# original data\n",
+    "fig = data.plot()\n",
+    "\n",
+    "original_image: CWLFilePathOutput = 'original_data.png'\n",
+    "fig.figure.savefig(original_image)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# transform data\n",
+    "data.sort_values(by='Y', ascending=False, inplace=True, ignore_index=True)\n",
+    "fig = data.plot()\n",
+    "\n",
+    "after_transform_data: CWLFilePathOutput = 'new_data.png'\n",
+    "fig.figure.savefig(after_transform_data)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "messages_filename = 'messages.txt'\n",
+    "with open(messages_filename, 'w') as f:\n",
+    "    f.write(' '.join(messages))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if optional_message is not None:\n",
+    "    print(optional_message)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.10"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 1
+}

tests/test_system_tests.py

@@ -9,12 +9,13 @@
 import pkg_resources
 import yaml
 from cwltool.context import RuntimeContext
-
+from cwltool.errors import WorkflowException
 
 class TestConsoleScripts(TestCase):
     maxDiff = None
     here = os.path.abspath(os.path.dirname(__file__))
     repo_like_dir = os.path.join(here, 'repo-like')
+    repo_like_dir_imports = os.path.join(here, 'repo-like-with-imports')
 
     @skipIf("TRAVIS_IGNORE_DOCKER" in os.environ and os.environ["TRAVIS_IGNORE_DOCKER"] == "true",
             "Skipping this test on Travis CI.")
@@ -56,6 +57,51 @@ def test_repo2cwl(self):
         self.assertEqual("hello test !!!", message)
         shutil.rmtree(output_dir)
 
+    @skipIf("TRAVIS_IGNORE_DOCKER" in os.environ and os.environ["TRAVIS_IGNORE_DOCKER"] == "true",
+            "Skipping this test on Travis CI.")
+    def test_repo2cwl_with_imports(self):
+        output_dir = tempfile.mkdtemp()
+        print(f'output directory:\t{output_dir}')
+        repo2cwl = pkg_resources.load_entry_point('ipython2cwl', 'console_scripts', 'jupyter-repo2cwl')
+        self.assertEqual(
+            0,
+            repo2cwl(['-o', output_dir, self.repo_like_dir_imports])
+        )
+        self.assertListEqual(['example1_import.cwl'], [f for f in os.listdir(output_dir) if not f.startswith('.')])
+
+        with open(os.path.join(output_dir, 'example1_import.cwl')) as f:
+            print('workflow file')
+            print(20 * '=')
+            print(f.read())
+            print(20 * '=')
+
+        runtime_context = RuntimeContext()
+        runtime_context.outdir = output_dir
+        runtime_context.basedir = output_dir
+        runtime_context.default_stdout = DEVNULL
+        runtime_context.default_stderr = DEVNULL
+        fac = cwltool.factory.Factory(runtime_context=runtime_context)
+
+        try:
+            example1_tool = fac.make(os.path.join(output_dir, 'example1_import.cwl'))
+        except WorkflowException:
+            self.fail("Execution failed")
+
+
+        result = example1_tool(
+            datafilename={
+                'class': 'File', 'location': os.path.join(self.repo_like_dir, 'data.yaml')
+            },
+            messages=["hello", "test", "!!!"]
+        )
+        with open(result['results_filename']['location'][7:]) as f:
+            new_data = yaml.safe_load(f)
+        self.assertDictEqual({'entry1': 2, 'entry2': 'foo', 'entry3': 'bar'}, new_data)
+        with open(result['messages_outputs']['location'][7:]) as f:
+            message = f.read()
+        self.assertEqual("hello test !!!", message)
+        shutil.rmtree(output_dir)
+
     def test_repo2cwl_output_dir_does_not_exists(self):
         random_dir_name = str(uuid.uuid4())
         repo2cwl = pkg_resources.load_entry_point('ipython2cwl', 'console_scripts', 'jupyter-repo2cwl')