diff --git a/ipython2cwl/repo2cwl.py b/ipython2cwl/repo2cwl.py index 78a89a2..d1967d5 100644 --- a/ipython2cwl/repo2cwl.py +++ b/ipython2cwl/repo2cwl.py @@ -23,7 +23,8 @@ def _get_notebook_paths_from_dir(dir_path: str): notebooks_paths = [] - for path, _, files in os.walk(dir_path): + for path, dirs, files in os.walk(dir_path): + dirs[:] = [d for d in dirs if d not in "cwl"] for name in files: if name.endswith('.ipynb'): notebooks_paths.append(os.path.join(path, name)) @@ -40,16 +41,14 @@ def _store_jn_as_script(notebook_path: str, git_directory_absolute_path: str, bi if len(converter._variables) == 0: logger.info(f"Notebook {notebook_path} does not contains typing annotations. skipping...") return None, None - script_relative_path = os.path.relpath(notebook_path, git_directory_absolute_path)[:-6] - script_relative_parent_directories = script_relative_path.split(os.sep) - if len(script_relative_parent_directories) > 1: - script_absolute_name = os.path.join(bin_absolute_path, os.sep.join(script_relative_parent_directories[:-1])) - os.makedirs( - script_absolute_name, - exist_ok=True) - script_absolute_name = os.path.join(script_absolute_name, os.path.basename(script_relative_path)) - else: - script_absolute_name = os.path.join(bin_absolute_path, script_relative_path) + + #change the extension from ipynb to nothing + notebook_absolute = Path(notebook_path) + notebook_relative_to_git = notebook_absolute.relative_to(git_directory_absolute_path) + notebook_name_without_extension = notebook_absolute.stem + script_absolute = Path(bin_absolute_path) / notebook_relative_to_git.parent / notebook_name_without_extension + #write the script, return it relative to + script_relative_to_git = script_absolute.relative_to(bin_absolute_path) script = os.linesep.join([ '#!/usr/bin/env ipython', '"""', @@ -59,13 +58,12 @@ def _store_jn_as_script(notebook_path: str, git_directory_absolute_path: str, bi '"""\n\n', converter._wrap_script_to_method(converter._tree, converter._variables) ]) - with open(script_absolute_name, 'w') as fd: + with open(script_absolute, 'w') as fd: fd.write(script) tool = converter.cwl_command_line_tool(image_id) - in_git_dir_script_file = os.path.join(bin_absolute_path, script_relative_path) - tool_st = os.stat(in_git_dir_script_file) - os.chmod(in_git_dir_script_file, tool_st.st_mode | stat.S_IEXEC) - return tool, script_relative_path + tool_st = os.stat(script_absolute) + os.chmod(script_absolute, tool_st.st_mode | stat.S_IEXEC) + return tool, str(script_relative_to_git) def existing_path(path_str: str): @@ -126,7 +124,8 @@ def repo2cwl(argv: Optional[List[str]] = None) -> int: image_id, cwl_tools = _repo2cwl(local_git) logger.info(f'Generated image id: {image_id}') for tool in cwl_tools: - base_command_script_name = f'{tool["baseCommand"][len("/app/cwl/bin/"):].replace("/", "_")}.cwl' + script_name_path = Path(tool["baseCommand"]).stem + base_command_script_name = f"""{str(script_name_path)}.cwl""" tool_filename = str(output_directory.joinpath(base_command_script_name)) with open(tool_filename, 'w') as f: logger.info(f'Creating CWL command line tool: {tool_filename}') @@ -148,9 +147,11 @@ def _repo2cwl(git_directory_path: Repo) -> Tuple[str, List[Dict]]: r2d.target_repo_dir = os.path.join(os.path.sep, 'app') r2d.repo = git_directory_path.tree().abspath bin_path = os.path.join(r2d.repo, 'cwl', 'bin') - os.makedirs(bin_path, exist_ok=True) - notebooks_paths = _get_notebook_paths_from_dir(r2d.repo) + shutil.copytree(r2d.repo, bin_path) + notebooks_paths = _get_notebook_paths_from_dir(r2d.repo) + logger.info(notebooks_paths) + tools = [] for notebook in notebooks_paths: cwl_command_line_tool, script_name = _store_jn_as_script( diff --git a/tests/repo-like-with-imports/example1_import.ipynb b/tests/repo-like-with-imports/example1_import.ipynb new file mode 100644 index 0000000..343bc80 --- /dev/null +++ b/tests/repo-like-with-imports/example1_import.ipynb @@ -0,0 +1,75 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ipython2cwl.iotypes import CWLFilePathInput, CWLStringInput, CWLFilePathOutput\n", + "from typing import List\n", + "import yaml\n", + "import os\n", + "\n", + "import simple_import" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "datafilename: CWLFilePathInput = 'data.yaml'\n", + "messages: List[CWLStringInput] = ['hello', 'world']\n", + "with open(datafilename) as fd: \n", + " data = yaml.safe_load(fd)\n", + "print(data)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "data['entry1'] += 1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "results_filename: CWLFilePathOutput = 'result.yaml'\n", + "with open(results_filename, 'w') as fd:\n", + " yaml.safe_dump(data, fd)\n", + "messages_outputs: CWLFilePathOutput = 'messages.txt'\n", + "with open(messages_outputs, 'w') as f:\n", + " f.write(' '.join(messages))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/tests/repo-like-with-imports/requirements.txt b/tests/repo-like-with-imports/requirements.txt new file mode 100644 index 0000000..7a997b5 --- /dev/null +++ b/tests/repo-like-with-imports/requirements.txt @@ -0,0 +1 @@ +PyYAML==5.3.1 diff --git a/tests/repo-like-with-imports/simple_import.py b/tests/repo-like-with-imports/simple_import.py new file mode 100644 index 0000000..970bc6c --- /dev/null +++ b/tests/repo-like-with-imports/simple_import.py @@ -0,0 +1,2 @@ +def hi(): + print("hi") \ No newline at end of file diff --git a/tests/simple_import.ipynb b/tests/simple_import.ipynb new file mode 100644 index 0000000..22067dd --- /dev/null +++ b/tests/simple_import.ipynb @@ -0,0 +1,97 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "from typing import List, Optional\n", + "import matplotlib\n", + "from ipython2cwl.iotypes import CWLFilePathInput, CWLFilePathOutput, CWLStringInput" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dataset: CWLFilePathInput = './data/data.csv'\n", + "messages: List[CWLStringInput] = ['hello', 'world']\n", + "optional_message: Optional[CWLStringInput] = \"Hello from optional\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "data = pd.read_csv(dataset)\n", + "# original data\n", + "fig = data.plot()\n", + "\n", + "original_image: CWLFilePathOutput = 'original_data.png'\n", + "fig.figure.savefig(original_image)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# transform data\n", + "data.sort_values(by='Y', ascending=False, inplace=True, ignore_index=True)\n", + "fig = data.plot()\n", + "\n", + "after_transform_data: CWLFilePathOutput = 'new_data.png'\n", + "fig.figure.savefig(after_transform_data)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "messages_filename = 'messages.txt'\n", + "with open(messages_filename, 'w') as f:\n", + " f.write(' '.join(messages))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if optional_message is not None:\n", + " print(optional_message)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} \ No newline at end of file diff --git a/tests/test_system_tests.py b/tests/test_system_tests.py index 2ca482b..2e529af 100644 --- a/tests/test_system_tests.py +++ b/tests/test_system_tests.py @@ -9,12 +9,13 @@ import pkg_resources import yaml from cwltool.context import RuntimeContext - +from cwltool.errors import WorkflowException class TestConsoleScripts(TestCase): maxDiff = None here = os.path.abspath(os.path.dirname(__file__)) repo_like_dir = os.path.join(here, 'repo-like') + repo_like_dir_imports = os.path.join(here, 'repo-like-with-imports') @skipIf("TRAVIS_IGNORE_DOCKER" in os.environ and os.environ["TRAVIS_IGNORE_DOCKER"] == "true", "Skipping this test on Travis CI.") @@ -56,6 +57,51 @@ def test_repo2cwl(self): self.assertEqual("hello test !!!", message) shutil.rmtree(output_dir) + @skipIf("TRAVIS_IGNORE_DOCKER" in os.environ and os.environ["TRAVIS_IGNORE_DOCKER"] == "true", + "Skipping this test on Travis CI.") + def test_repo2cwl_with_imports(self): + output_dir = tempfile.mkdtemp() + print(f'output directory:\t{output_dir}') + repo2cwl = pkg_resources.load_entry_point('ipython2cwl', 'console_scripts', 'jupyter-repo2cwl') + self.assertEqual( + 0, + repo2cwl(['-o', output_dir, self.repo_like_dir_imports]) + ) + self.assertListEqual(['example1_import.cwl'], [f for f in os.listdir(output_dir) if not f.startswith('.')]) + + with open(os.path.join(output_dir, 'example1_import.cwl')) as f: + print('workflow file') + print(20 * '=') + print(f.read()) + print(20 * '=') + + runtime_context = RuntimeContext() + runtime_context.outdir = output_dir + runtime_context.basedir = output_dir + runtime_context.default_stdout = DEVNULL + runtime_context.default_stderr = DEVNULL + fac = cwltool.factory.Factory(runtime_context=runtime_context) + + try: + example1_tool = fac.make(os.path.join(output_dir, 'example1_import.cwl')) + except WorkflowException: + self.fail("Execution failed") + + + result = example1_tool( + datafilename={ + 'class': 'File', 'location': os.path.join(self.repo_like_dir, 'data.yaml') + }, + messages=["hello", "test", "!!!"] + ) + with open(result['results_filename']['location'][7:]) as f: + new_data = yaml.safe_load(f) + self.assertDictEqual({'entry1': 2, 'entry2': 'foo', 'entry3': 'bar'}, new_data) + with open(result['messages_outputs']['location'][7:]) as f: + message = f.read() + self.assertEqual("hello test !!!", message) + shutil.rmtree(output_dir) + def test_repo2cwl_output_dir_does_not_exists(self): random_dir_name = str(uuid.uuid4()) repo2cwl = pkg_resources.load_entry_point('ipython2cwl', 'console_scripts', 'jupyter-repo2cwl')