From cd662cb4c6b0dd525e66d08e8f99755339a8e142 Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Fri, 7 May 2021 10:34:38 -0400 Subject: [PATCH 01/19] fix: add test files --- tests/simple_import.ipynb | 98 +++++++++++++++++++++++++++++++++++++++ tests/simple_import.py | 2 + 2 files changed, 100 insertions(+) create mode 100644 tests/simple_import.ipynb create mode 100644 tests/simple_import.py diff --git a/tests/simple_import.ipynb b/tests/simple_import.ipynb new file mode 100644 index 0000000..8f2c212 --- /dev/null +++ b/tests/simple_import.ipynb @@ -0,0 +1,98 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from simple_import import hi\n", + "import pandas as pd\n", + "from typing import List, Optional\n", + "import matplotlib\n", + "from ipython2cwl.iotypes import CWLFilePathInput, CWLFilePathOutput, CWLStringInput" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "dataset: CWLFilePathInput = './data/data.csv'\n", + "messages: List[CWLStringInput] = ['hello', 'world']\n", + "optional_message: Optional[CWLStringInput] = \"Hello from optional\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "data = pd.read_csv(dataset)\n", + "# original data\n", + "fig = data.plot()\n", + "\n", + "original_image: CWLFilePathOutput = 'original_data.png'\n", + "fig.figure.savefig(original_image)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# transform data\n", + "data.sort_values(by='Y', ascending=False, inplace=True, ignore_index=True)\n", + "fig = data.plot()\n", + "\n", + "after_transform_data: CWLFilePathOutput = 'new_data.png'\n", + "fig.figure.savefig(after_transform_data)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "messages_filename = 'messages.txt'\n", + "with open(messages_filename, 'w') as f:\n", + " f.write(' '.join(messages))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "if optional_message is not None:\n", + " print(optional_message)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" + } + }, + "nbformat": 4, + "nbformat_minor": 1 +} \ No newline at end of file diff --git a/tests/simple_import.py b/tests/simple_import.py new file mode 100644 index 0000000..970bc6c --- /dev/null +++ b/tests/simple_import.py @@ -0,0 +1,2 @@ +def hi(): + print("hi") \ No newline at end of file From d4ddb9be2dc334c4771221de9aa6ccc69905009c Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Fri, 7 May 2021 11:05:51 -0400 Subject: [PATCH 02/19] fix: add new function to test a new notebook --- tests/simple_import.ipynb | 1 - tests/test_ipython2cwl_from_repo.py | 104 ++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 1 deletion(-) diff --git a/tests/simple_import.ipynb b/tests/simple_import.ipynb index 8f2c212..22067dd 100644 --- a/tests/simple_import.ipynb +++ b/tests/simple_import.ipynb @@ -6,7 +6,6 @@ "metadata": {}, "outputs": [], "source": [ - "from simple_import import hi\n", "import pandas as pd\n", "from typing import List, Optional\n", "import matplotlib\n", diff --git a/tests/test_ipython2cwl_from_repo.py b/tests/test_ipython2cwl_from_repo.py index a26ab9c..56cf19b 100644 --- a/tests/test_ipython2cwl_from_repo.py +++ b/tests/test_ipython2cwl_from_repo.py @@ -142,3 +142,107 @@ def test_docker_build(self): self.assertListEqual(base_commands, ['/app/cwl/bin/simple', '/app/cwl/bin/subdir/simple']) script = docker_client.containers.run(dockerfile_image_id, '/app/cwl/bin/subdir/simple', entrypoint='/bin/cat') self.assertIn('fig.figure.savefig(after_transform_data)', script.decode()) + + def test_notebook_with_imports(self): + # setup a simple git repo + git_dir = tempfile.mkdtemp() + jn_repo = Repo.init(git_dir) + shutil.copy( + os.path.join(self.here, 'simple_import.ipynb'), + os.path.join(git_dir, 'simple_import.ipynb'), + ) + with open(os.path.join(git_dir, 'requirements.txt'), 'w') as f: + f.write('pandas\n') + f.write('matplotlib\n') + jn_repo.index.add('simple_import.ipynb') + jn_repo.index.add('requirements.txt') + jn_repo.index.commit("initial commit") + + print(git_dir) + + dockerfile_image_id, cwl_tool = _repo2cwl(jn_repo) + self.assertEqual(1, len(cwl_tool)) + docker_client = docker.from_env() + script = docker_client.containers.run(dockerfile_image_id, '/app/cwl/bin/simple_import', entrypoint='/bin/cat') + self.assertIn('fig.figure.savefig(after_transform_data)', script.decode()) + messages_array_arg_line = ast.parse( + [line.strip() for line in script.decode().splitlines() if '--messages' in line][-1] + ) + self.assertEqual( + '+', # nargs = '+' + [k.value.s for k in messages_array_arg_line.body[0].value.keywords if k.arg == 'nargs'][0] + ) + self.assertEqual( + 'str', # type = 'str' + [k.value.id for k in messages_array_arg_line.body[0].value.keywords if k.arg == 'type'][0] + ) + + script_tree = ast.parse(script.decode()) + optional_expression = [x for x in script_tree.body[-1].body if + isinstance(x, ast.Expr) and isinstance(x.value, ast.Call) and len(x.value.args) > 0 and + x.value.args[0].s == '--optional_message'][0] + self.assertEqual( + False, + [k.value for k in optional_expression.value.keywords if k.arg == 'required'][0].value + ) + self.assertEqual( + None, + [k.value for k in optional_expression.value.keywords if k.arg == 'default'][0].value + ) + self.assertDictEqual( + { + 'cwlVersion': "v1.1", + 'class': 'CommandLineTool', + 'baseCommand': '/app/cwl/bin/simple_import', + 'hints': { + 'DockerRequirement': {'dockerImageId': dockerfile_image_id} + }, + 'arguments': ['--'], + 'inputs': { + 'dataset': { + 'type': 'File', + 'inputBinding': { + 'prefix': '--dataset' + } + }, + 'messages': { + 'type': 'string[]', + 'inputBinding': { + 'prefix': '--messages' + } + }, + 'optional_message': { + 'type': 'string?', + 'inputBinding': { + 'prefix': '--optional_message' + } + } + }, + 'outputs': { + 'original_image': { + 'type': 'File', + 'outputBinding': { + 'glob': 'original_data.png' + } + }, + 'after_transform_data': { + 'type': 'File', + 'outputBinding': { + 'glob': 'new_data.png' + } + } + }, + }, + cwl_tool[0] + ) + cwl = StringIO() + yaml.safe_dump(cwl_tool[0], cwl) + cwl_code = cwl.getvalue() + print(cwl_code) + + #test + dockerfile_image_id, new_cwl_tool = _repo2cwl(jn_repo) + self.assertEqual(1, len(new_cwl_tool)) + cwl_tool[0]['hints']['DockerRequirement'].pop('dockerImageId') + new_cwl_tool[0]['hints']['DockerRequirement'].pop('dockerImageId') + self.assertListEqual(cwl_tool, new_cwl_tool) \ No newline at end of file From 916865e4d80ec032fa31930706bcded11baa2fb5 Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Fri, 7 May 2021 11:46:03 -0400 Subject: [PATCH 03/19] fix: add tests as directory --- tests/repo-like-with-imports/example1.ipynb | 75 +++++++++++++ tests/repo-like-with-imports/requirements.txt | 1 + .../simple_import.py | 0 tests/test_ipython2cwl_from_repo.py | 104 ------------------ tests/test_system_tests.py | 41 +++++++ 5 files changed, 117 insertions(+), 104 deletions(-) create mode 100644 tests/repo-like-with-imports/example1.ipynb create mode 100644 tests/repo-like-with-imports/requirements.txt rename tests/{ => repo-like-with-imports}/simple_import.py (100%) diff --git a/tests/repo-like-with-imports/example1.ipynb b/tests/repo-like-with-imports/example1.ipynb new file mode 100644 index 0000000..343bc80 --- /dev/null +++ b/tests/repo-like-with-imports/example1.ipynb @@ -0,0 +1,75 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ipython2cwl.iotypes import CWLFilePathInput, CWLStringInput, CWLFilePathOutput\n", + "from typing import List\n", + "import yaml\n", + "import os\n", + "\n", + "import simple_import" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "datafilename: CWLFilePathInput = 'data.yaml'\n", + "messages: List[CWLStringInput] = ['hello', 'world']\n", + "with open(datafilename) as fd: \n", + " data = yaml.safe_load(fd)\n", + "print(data)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "data['entry1'] += 1" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "results_filename: CWLFilePathOutput = 'result.yaml'\n", + "with open(results_filename, 'w') as fd:\n", + " yaml.safe_dump(data, fd)\n", + "messages_outputs: CWLFilePathOutput = 'messages.txt'\n", + "with open(messages_outputs, 'w') as f:\n", + " f.write(' '.join(messages))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/tests/repo-like-with-imports/requirements.txt b/tests/repo-like-with-imports/requirements.txt new file mode 100644 index 0000000..7a997b5 --- /dev/null +++ b/tests/repo-like-with-imports/requirements.txt @@ -0,0 +1 @@ +PyYAML==5.3.1 diff --git a/tests/simple_import.py b/tests/repo-like-with-imports/simple_import.py similarity index 100% rename from tests/simple_import.py rename to tests/repo-like-with-imports/simple_import.py diff --git a/tests/test_ipython2cwl_from_repo.py b/tests/test_ipython2cwl_from_repo.py index 56cf19b..a26ab9c 100644 --- a/tests/test_ipython2cwl_from_repo.py +++ b/tests/test_ipython2cwl_from_repo.py @@ -142,107 +142,3 @@ def test_docker_build(self): self.assertListEqual(base_commands, ['/app/cwl/bin/simple', '/app/cwl/bin/subdir/simple']) script = docker_client.containers.run(dockerfile_image_id, '/app/cwl/bin/subdir/simple', entrypoint='/bin/cat') self.assertIn('fig.figure.savefig(after_transform_data)', script.decode()) - - def test_notebook_with_imports(self): - # setup a simple git repo - git_dir = tempfile.mkdtemp() - jn_repo = Repo.init(git_dir) - shutil.copy( - os.path.join(self.here, 'simple_import.ipynb'), - os.path.join(git_dir, 'simple_import.ipynb'), - ) - with open(os.path.join(git_dir, 'requirements.txt'), 'w') as f: - f.write('pandas\n') - f.write('matplotlib\n') - jn_repo.index.add('simple_import.ipynb') - jn_repo.index.add('requirements.txt') - jn_repo.index.commit("initial commit") - - print(git_dir) - - dockerfile_image_id, cwl_tool = _repo2cwl(jn_repo) - self.assertEqual(1, len(cwl_tool)) - docker_client = docker.from_env() - script = docker_client.containers.run(dockerfile_image_id, '/app/cwl/bin/simple_import', entrypoint='/bin/cat') - self.assertIn('fig.figure.savefig(after_transform_data)', script.decode()) - messages_array_arg_line = ast.parse( - [line.strip() for line in script.decode().splitlines() if '--messages' in line][-1] - ) - self.assertEqual( - '+', # nargs = '+' - [k.value.s for k in messages_array_arg_line.body[0].value.keywords if k.arg == 'nargs'][0] - ) - self.assertEqual( - 'str', # type = 'str' - [k.value.id for k in messages_array_arg_line.body[0].value.keywords if k.arg == 'type'][0] - ) - - script_tree = ast.parse(script.decode()) - optional_expression = [x for x in script_tree.body[-1].body if - isinstance(x, ast.Expr) and isinstance(x.value, ast.Call) and len(x.value.args) > 0 and - x.value.args[0].s == '--optional_message'][0] - self.assertEqual( - False, - [k.value for k in optional_expression.value.keywords if k.arg == 'required'][0].value - ) - self.assertEqual( - None, - [k.value for k in optional_expression.value.keywords if k.arg == 'default'][0].value - ) - self.assertDictEqual( - { - 'cwlVersion': "v1.1", - 'class': 'CommandLineTool', - 'baseCommand': '/app/cwl/bin/simple_import', - 'hints': { - 'DockerRequirement': {'dockerImageId': dockerfile_image_id} - }, - 'arguments': ['--'], - 'inputs': { - 'dataset': { - 'type': 'File', - 'inputBinding': { - 'prefix': '--dataset' - } - }, - 'messages': { - 'type': 'string[]', - 'inputBinding': { - 'prefix': '--messages' - } - }, - 'optional_message': { - 'type': 'string?', - 'inputBinding': { - 'prefix': '--optional_message' - } - } - }, - 'outputs': { - 'original_image': { - 'type': 'File', - 'outputBinding': { - 'glob': 'original_data.png' - } - }, - 'after_transform_data': { - 'type': 'File', - 'outputBinding': { - 'glob': 'new_data.png' - } - } - }, - }, - cwl_tool[0] - ) - cwl = StringIO() - yaml.safe_dump(cwl_tool[0], cwl) - cwl_code = cwl.getvalue() - print(cwl_code) - - #test - dockerfile_image_id, new_cwl_tool = _repo2cwl(jn_repo) - self.assertEqual(1, len(new_cwl_tool)) - cwl_tool[0]['hints']['DockerRequirement'].pop('dockerImageId') - new_cwl_tool[0]['hints']['DockerRequirement'].pop('dockerImageId') - self.assertListEqual(cwl_tool, new_cwl_tool) \ No newline at end of file diff --git a/tests/test_system_tests.py b/tests/test_system_tests.py index 2ca482b..52f49ce 100644 --- a/tests/test_system_tests.py +++ b/tests/test_system_tests.py @@ -15,6 +15,7 @@ class TestConsoleScripts(TestCase): maxDiff = None here = os.path.abspath(os.path.dirname(__file__)) repo_like_dir = os.path.join(here, 'repo-like') + repo_like_dir_imports = os.path.join(here, 'repo-like-with-imports') @skipIf("TRAVIS_IGNORE_DOCKER" in os.environ and os.environ["TRAVIS_IGNORE_DOCKER"] == "true", "Skipping this test on Travis CI.") @@ -56,6 +57,46 @@ def test_repo2cwl(self): self.assertEqual("hello test !!!", message) shutil.rmtree(output_dir) + @skipIf("TRAVIS_IGNORE_DOCKER" in os.environ and os.environ["TRAVIS_IGNORE_DOCKER"] == "true", + "Skipping this test on Travis CI.") + def test_repo2cwl_with_imports(self): + output_dir = tempfile.mkdtemp() + print(f'output directory:\t{output_dir}') + repo2cwl = pkg_resources.load_entry_point('ipython2cwl', 'console_scripts', 'jupyter-repo2cwl') + self.assertEqual( + 0, + repo2cwl(['-o', output_dir, self.repo_like_dir_imports]) + ) + self.assertListEqual(['example1.cwl'], [f for f in os.listdir(output_dir) if not f.startswith('.')]) + + with open(os.path.join(output_dir, 'example1.cwl')) as f: + print('workflow file') + print(20 * '=') + print(f.read()) + print(20 * '=') + + runtime_context = RuntimeContext() + runtime_context.outdir = output_dir + runtime_context.basedir = output_dir + runtime_context.default_stdout = DEVNULL + runtime_context.default_stderr = DEVNULL + fac = cwltool.factory.Factory(runtime_context=runtime_context) + + example1_tool = fac.make(os.path.join(output_dir, 'example1.cwl')) + result = example1_tool( + datafilename={ + 'class': 'File', 'location': os.path.join(self.repo_like_dir, 'data.yaml') + }, + messages=["hello", "test", "!!!"] + ) + with open(result['results_filename']['location'][7:]) as f: + new_data = yaml.safe_load(f) + self.assertDictEqual({'entry1': 2, 'entry2': 'foo', 'entry3': 'bar'}, new_data) + with open(result['messages_outputs']['location'][7:]) as f: + message = f.read() + self.assertEqual("hello test !!!", message) + shutil.rmtree(output_dir) + def test_repo2cwl_output_dir_does_not_exists(self): random_dir_name = str(uuid.uuid4()) repo2cwl = pkg_resources.load_entry_point('ipython2cwl', 'console_scripts', 'jupyter-repo2cwl') From ea89d7cf19716025e2900db505b2b1e517d7bc56 Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Fri, 7 May 2021 12:14:11 -0400 Subject: [PATCH 04/19] fix: check error when cwltool fails --- tests/test_system_tests.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/test_system_tests.py b/tests/test_system_tests.py index 52f49ce..d6bb26c 100644 --- a/tests/test_system_tests.py +++ b/tests/test_system_tests.py @@ -9,7 +9,7 @@ import pkg_resources import yaml from cwltool.context import RuntimeContext - +from cwltool.errors import WorkflowException class TestConsoleScripts(TestCase): maxDiff = None @@ -82,7 +82,12 @@ def test_repo2cwl_with_imports(self): runtime_context.default_stderr = DEVNULL fac = cwltool.factory.Factory(runtime_context=runtime_context) - example1_tool = fac.make(os.path.join(output_dir, 'example1.cwl')) + try: + example1_tool = fac.make(os.path.join(output_dir, 'example1.cwl')) + except WorkflowException: + self.fail("Execution failed") + + result = example1_tool( datafilename={ 'class': 'File', 'location': os.path.join(self.repo_like_dir, 'data.yaml') From fef4ce44917cd4f0bef9a1e98c39a19864ff59bd Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Fri, 7 May 2021 14:39:33 -0400 Subject: [PATCH 05/19] fix: copy the folder into the bin --- ipython2cwl/repo2cwl.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ipython2cwl/repo2cwl.py b/ipython2cwl/repo2cwl.py index 78a89a2..8b34337 100644 --- a/ipython2cwl/repo2cwl.py +++ b/ipython2cwl/repo2cwl.py @@ -40,13 +40,12 @@ def _store_jn_as_script(notebook_path: str, git_directory_absolute_path: str, bi if len(converter._variables) == 0: logger.info(f"Notebook {notebook_path} does not contains typing annotations. skipping...") return None, None + + script_relative_path = os.path.relpath(notebook_path, git_directory_absolute_path)[:-6] script_relative_parent_directories = script_relative_path.split(os.sep) if len(script_relative_parent_directories) > 1: script_absolute_name = os.path.join(bin_absolute_path, os.sep.join(script_relative_parent_directories[:-1])) - os.makedirs( - script_absolute_name, - exist_ok=True) script_absolute_name = os.path.join(script_absolute_name, os.path.basename(script_relative_path)) else: script_absolute_name = os.path.join(bin_absolute_path, script_relative_path) @@ -148,7 +147,7 @@ def _repo2cwl(git_directory_path: Repo) -> Tuple[str, List[Dict]]: r2d.target_repo_dir = os.path.join(os.path.sep, 'app') r2d.repo = git_directory_path.tree().abspath bin_path = os.path.join(r2d.repo, 'cwl', 'bin') - os.makedirs(bin_path, exist_ok=True) + shutil.copytree(r2d.repo, bin_path) notebooks_paths = _get_notebook_paths_from_dir(r2d.repo) tools = [] From 676cb6068385e2628599cd38cec554fabd0f1b80 Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Fri, 7 May 2021 14:57:07 -0400 Subject: [PATCH 06/19] fix: refactoring the bin directory --- ipython2cwl/repo2cwl.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/ipython2cwl/repo2cwl.py b/ipython2cwl/repo2cwl.py index 8b34337..574badc 100644 --- a/ipython2cwl/repo2cwl.py +++ b/ipython2cwl/repo2cwl.py @@ -41,14 +41,11 @@ def _store_jn_as_script(notebook_path: str, git_directory_absolute_path: str, bi logger.info(f"Notebook {notebook_path} does not contains typing annotations. skipping...") return None, None + #change the extension from ipynb to nothing + script_name = Path(notebook_path).stem + notebook_parent_relative = Path(notebook_path).relative_to(git_directory_absolute_path).parent + script_absolute_path = Path(bin_absolute_path) / notebook_parent_relative / script_name - script_relative_path = os.path.relpath(notebook_path, git_directory_absolute_path)[:-6] - script_relative_parent_directories = script_relative_path.split(os.sep) - if len(script_relative_parent_directories) > 1: - script_absolute_name = os.path.join(bin_absolute_path, os.sep.join(script_relative_parent_directories[:-1])) - script_absolute_name = os.path.join(script_absolute_name, os.path.basename(script_relative_path)) - else: - script_absolute_name = os.path.join(bin_absolute_path, script_relative_path) script = os.linesep.join([ '#!/usr/bin/env ipython', '"""', @@ -58,13 +55,12 @@ def _store_jn_as_script(notebook_path: str, git_directory_absolute_path: str, bi '"""\n\n', converter._wrap_script_to_method(converter._tree, converter._variables) ]) - with open(script_absolute_name, 'w') as fd: + with open(script_absolute_path, 'w') as fd: fd.write(script) tool = converter.cwl_command_line_tool(image_id) - in_git_dir_script_file = os.path.join(bin_absolute_path, script_relative_path) - tool_st = os.stat(in_git_dir_script_file) - os.chmod(in_git_dir_script_file, tool_st.st_mode | stat.S_IEXEC) - return tool, script_relative_path + tool_st = os.stat(script_absolute_path) + os.chmod(script_absolute_path, tool_st.st_mode | stat.S_IEXEC) + return tool, script_absolute_path def existing_path(path_str: str): @@ -148,6 +144,7 @@ def _repo2cwl(git_directory_path: Repo) -> Tuple[str, List[Dict]]: r2d.repo = git_directory_path.tree().abspath bin_path = os.path.join(r2d.repo, 'cwl', 'bin') shutil.copytree(r2d.repo, bin_path) + notebooks_paths = _get_notebook_paths_from_dir(r2d.repo) tools = [] @@ -160,7 +157,7 @@ def _repo2cwl(git_directory_path: Repo) -> Tuple[str, List[Dict]]: ) if cwl_command_line_tool is None or script_name is None: continue - cwl_command_line_tool['baseCommand'] = os.path.join('/app', 'cwl', 'bin', script_name) + cwl_command_line_tool['baseCommand'] = script_name tools.append(cwl_command_line_tool) git_directory_path.index.commit("auto-commit") From c8e726d671e60ffb6eb0e6eb98144bcd8314d6a5 Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Fri, 7 May 2021 14:59:00 -0400 Subject: [PATCH 07/19] fix: logging notebook --- ipython2cwl/repo2cwl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ipython2cwl/repo2cwl.py b/ipython2cwl/repo2cwl.py index 574badc..99d9ff0 100644 --- a/ipython2cwl/repo2cwl.py +++ b/ipython2cwl/repo2cwl.py @@ -146,7 +146,8 @@ def _repo2cwl(git_directory_path: Repo) -> Tuple[str, List[Dict]]: shutil.copytree(r2d.repo, bin_path) notebooks_paths = _get_notebook_paths_from_dir(r2d.repo) - + logging.info(notebook_paths) + tools = [] for notebook in notebooks_paths: cwl_command_line_tool, script_name = _store_jn_as_script( From 9dfae0960a4facd11531c2cc46ff304223f2be78 Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Fri, 7 May 2021 14:59:39 -0400 Subject: [PATCH 08/19] fix: typo --- ipython2cwl/repo2cwl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipython2cwl/repo2cwl.py b/ipython2cwl/repo2cwl.py index 99d9ff0..fe7e783 100644 --- a/ipython2cwl/repo2cwl.py +++ b/ipython2cwl/repo2cwl.py @@ -146,7 +146,7 @@ def _repo2cwl(git_directory_path: Repo) -> Tuple[str, List[Dict]]: shutil.copytree(r2d.repo, bin_path) notebooks_paths = _get_notebook_paths_from_dir(r2d.repo) - logging.info(notebook_paths) + logging.info(notebooks_paths) tools = [] for notebook in notebooks_paths: From e69a56b95a8785441dafe2ce0b5953c260afc7aa Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Fri, 7 May 2021 15:00:43 -0400 Subject: [PATCH 09/19] fix: log --- ipython2cwl/repo2cwl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipython2cwl/repo2cwl.py b/ipython2cwl/repo2cwl.py index fe7e783..9c50153 100644 --- a/ipython2cwl/repo2cwl.py +++ b/ipython2cwl/repo2cwl.py @@ -146,7 +146,7 @@ def _repo2cwl(git_directory_path: Repo) -> Tuple[str, List[Dict]]: shutil.copytree(r2d.repo, bin_path) notebooks_paths = _get_notebook_paths_from_dir(r2d.repo) - logging.info(notebooks_paths) + logger.info(notebooks_paths) tools = [] for notebook in notebooks_paths: From 03cdeb9911c97a0c243bbbcf2f7b3868dcbfb2d6 Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Fri, 7 May 2021 15:02:22 -0400 Subject: [PATCH 10/19] fix: exclude cwl --- ipython2cwl/repo2cwl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ipython2cwl/repo2cwl.py b/ipython2cwl/repo2cwl.py index 9c50153..2f2f83f 100644 --- a/ipython2cwl/repo2cwl.py +++ b/ipython2cwl/repo2cwl.py @@ -23,7 +23,8 @@ def _get_notebook_paths_from_dir(dir_path: str): notebooks_paths = [] - for path, _, files in os.walk(dir_path): + for path, dirs, files in os.walk(dir_path): + dirs[:] = [d for d in dirs if d not in "cwl"] for name in files: if name.endswith('.ipynb'): notebooks_paths.append(os.path.join(path, name)) From 762b8275d847c6de94648d23d10dd06b3d614a17 Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Fri, 7 May 2021 15:14:14 -0400 Subject: [PATCH 11/19] fix: clean name extraction --- ipython2cwl/repo2cwl.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ipython2cwl/repo2cwl.py b/ipython2cwl/repo2cwl.py index 2f2f83f..f8c24f6 100644 --- a/ipython2cwl/repo2cwl.py +++ b/ipython2cwl/repo2cwl.py @@ -122,7 +122,8 @@ def repo2cwl(argv: Optional[List[str]] = None) -> int: image_id, cwl_tools = _repo2cwl(local_git) logger.info(f'Generated image id: {image_id}') for tool in cwl_tools: - base_command_script_name = f'{tool["baseCommand"][len("/app/cwl/bin/"):].replace("/", "_")}.cwl' + script_name_path = Path(tool["baseCommand"]).stem + base_command_script_name = f"""{script_name_path}.cwl""" tool_filename = str(output_directory.joinpath(base_command_script_name)) with open(tool_filename, 'w') as f: logger.info(f'Creating CWL command line tool: {tool_filename}') From bf274e0bccd47ba10b062751a27d299f33fac67c Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Fri, 7 May 2021 15:15:49 -0400 Subject: [PATCH 12/19] fix: convert path to str --- ipython2cwl/repo2cwl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipython2cwl/repo2cwl.py b/ipython2cwl/repo2cwl.py index f8c24f6..757df6a 100644 --- a/ipython2cwl/repo2cwl.py +++ b/ipython2cwl/repo2cwl.py @@ -123,7 +123,7 @@ def repo2cwl(argv: Optional[List[str]] = None) -> int: logger.info(f'Generated image id: {image_id}') for tool in cwl_tools: script_name_path = Path(tool["baseCommand"]).stem - base_command_script_name = f"""{script_name_path}.cwl""" + base_command_script_name = f"""{str(script_name_path)}.cwl""" tool_filename = str(output_directory.joinpath(base_command_script_name)) with open(tool_filename, 'w') as f: logger.info(f'Creating CWL command line tool: {tool_filename}') From 7f96422c2f240af8bc27c02cdd7dc982e2e4a6f9 Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Fri, 7 May 2021 15:17:28 -0400 Subject: [PATCH 13/19] fix: return a str --- ipython2cwl/repo2cwl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipython2cwl/repo2cwl.py b/ipython2cwl/repo2cwl.py index 757df6a..ef33895 100644 --- a/ipython2cwl/repo2cwl.py +++ b/ipython2cwl/repo2cwl.py @@ -61,7 +61,7 @@ def _store_jn_as_script(notebook_path: str, git_directory_absolute_path: str, bi tool = converter.cwl_command_line_tool(image_id) tool_st = os.stat(script_absolute_path) os.chmod(script_absolute_path, tool_st.st_mode | stat.S_IEXEC) - return tool, script_absolute_path + return tool, str(script_absolute_path) def existing_path(path_str: str): From 15a29ebd18099704f39c496e9a4b0dac816ede1e Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Fri, 7 May 2021 16:01:34 -0400 Subject: [PATCH 14/19] fix: paths --- ipython2cwl/repo2cwl.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ipython2cwl/repo2cwl.py b/ipython2cwl/repo2cwl.py index ef33895..0d05ca5 100644 --- a/ipython2cwl/repo2cwl.py +++ b/ipython2cwl/repo2cwl.py @@ -43,10 +43,10 @@ def _store_jn_as_script(notebook_path: str, git_directory_absolute_path: str, bi return None, None #change the extension from ipynb to nothing - script_name = Path(notebook_path).stem - notebook_parent_relative = Path(notebook_path).relative_to(git_directory_absolute_path).parent - script_absolute_path = Path(bin_absolute_path) / notebook_parent_relative / script_name - + notebook_absolute = Path(notebook_path) + notebook_name_without_extension = notebook_absolute.stem + script_absolute = notebook.parent / f"""{notebook_name_without_extension}.py""" + script_relative_to_git = Path(notebook_path).relative_to(git_directory_absolute_path) script = os.linesep.join([ '#!/usr/bin/env ipython', '"""', @@ -56,12 +56,12 @@ def _store_jn_as_script(notebook_path: str, git_directory_absolute_path: str, bi '"""\n\n', converter._wrap_script_to_method(converter._tree, converter._variables) ]) - with open(script_absolute_path, 'w') as fd: + with open(script_absolute, 'w') as fd: fd.write(script) tool = converter.cwl_command_line_tool(image_id) - tool_st = os.stat(script_absolute_path) - os.chmod(script_absolute_path, tool_st.st_mode | stat.S_IEXEC) - return tool, str(script_absolute_path) + tool_st = os.stat(script_absolute) + os.chmod(script_absolute, tool_st.st_mode | stat.S_IEXEC) + return tool, str(script_relative_to_git) def existing_path(path_str: str): @@ -160,7 +160,7 @@ def _repo2cwl(git_directory_path: Repo) -> Tuple[str, List[Dict]]: ) if cwl_command_line_tool is None or script_name is None: continue - cwl_command_line_tool['baseCommand'] = script_name + cwl_command_line_tool['baseCommand'] = os.path.join('/app', 'cwl', 'bin', script_name) tools.append(cwl_command_line_tool) git_directory_path.index.commit("auto-commit") From a9816ff51e668cfe75935f56aca440bd25bab2eb Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Fri, 7 May 2021 16:05:04 -0400 Subject: [PATCH 15/19] fix: comments --- ipython2cwl/repo2cwl.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ipython2cwl/repo2cwl.py b/ipython2cwl/repo2cwl.py index 0d05ca5..08d1e31 100644 --- a/ipython2cwl/repo2cwl.py +++ b/ipython2cwl/repo2cwl.py @@ -45,8 +45,9 @@ def _store_jn_as_script(notebook_path: str, git_directory_absolute_path: str, bi #change the extension from ipynb to nothing notebook_absolute = Path(notebook_path) notebook_name_without_extension = notebook_absolute.stem - script_absolute = notebook.parent / f"""{notebook_name_without_extension}.py""" - script_relative_to_git = Path(notebook_path).relative_to(git_directory_absolute_path) + script_absolute = notebook_absolute.parent / notebook_name_without_extension + #write the script, return it relative to the repo git + script_relative_to_git = notebook_absolute.relative_to(git_directory_absolute_path) script = os.linesep.join([ '#!/usr/bin/env ipython', '"""', From 9c9994191d9aceaf795ead66f0930bace05d1ee5 Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Fri, 7 May 2021 16:06:06 -0400 Subject: [PATCH 16/19] fix: extract the basecommand from script not notebook --- ipython2cwl/repo2cwl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipython2cwl/repo2cwl.py b/ipython2cwl/repo2cwl.py index 08d1e31..023cc73 100644 --- a/ipython2cwl/repo2cwl.py +++ b/ipython2cwl/repo2cwl.py @@ -47,7 +47,7 @@ def _store_jn_as_script(notebook_path: str, git_directory_absolute_path: str, bi notebook_name_without_extension = notebook_absolute.stem script_absolute = notebook_absolute.parent / notebook_name_without_extension #write the script, return it relative to the repo git - script_relative_to_git = notebook_absolute.relative_to(git_directory_absolute_path) + script_relative_to_git = script_absolute.relative_to(git_directory_absolute_path) script = os.linesep.join([ '#!/usr/bin/env ipython', '"""', From 12ac3799d5087615f3b9ab86c31df2a2640a9d36 Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Fri, 7 May 2021 16:12:31 -0400 Subject: [PATCH 17/19] fix: relatives --- ipython2cwl/repo2cwl.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ipython2cwl/repo2cwl.py b/ipython2cwl/repo2cwl.py index 023cc73..839b5cc 100644 --- a/ipython2cwl/repo2cwl.py +++ b/ipython2cwl/repo2cwl.py @@ -44,10 +44,11 @@ def _store_jn_as_script(notebook_path: str, git_directory_absolute_path: str, bi #change the extension from ipynb to nothing notebook_absolute = Path(notebook_path) + notebook_relative_to_git = notebook_absolute.relative_to(git_directory_absolute_path) notebook_name_without_extension = notebook_absolute.stem - script_absolute = notebook_absolute.parent / notebook_name_without_extension - #write the script, return it relative to the repo git - script_relative_to_git = script_absolute.relative_to(git_directory_absolute_path) + script_absolute = Path(bin_absolute_path) / notebook_relative_to_git / notebook_name_without_extension + #write the script, return it relative to + script_relative_to_git = script_absolute.relative_to(bin_absolute_path) script = os.linesep.join([ '#!/usr/bin/env ipython', '"""', From 054f3523c5a8dbc00bdbd38d0d64e596751c7447 Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Fri, 7 May 2021 16:15:25 -0400 Subject: [PATCH 18/19] fix: parent --- ipython2cwl/repo2cwl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ipython2cwl/repo2cwl.py b/ipython2cwl/repo2cwl.py index 839b5cc..d1967d5 100644 --- a/ipython2cwl/repo2cwl.py +++ b/ipython2cwl/repo2cwl.py @@ -46,7 +46,7 @@ def _store_jn_as_script(notebook_path: str, git_directory_absolute_path: str, bi notebook_absolute = Path(notebook_path) notebook_relative_to_git = notebook_absolute.relative_to(git_directory_absolute_path) notebook_name_without_extension = notebook_absolute.stem - script_absolute = Path(bin_absolute_path) / notebook_relative_to_git / notebook_name_without_extension + script_absolute = Path(bin_absolute_path) / notebook_relative_to_git.parent / notebook_name_without_extension #write the script, return it relative to script_relative_to_git = script_absolute.relative_to(bin_absolute_path) script = os.linesep.join([ From e9479fc1c612db1c69a6384b27178cc305d4aa19 Mon Sep 17 00:00:00 2001 From: Maximiliano Osorio Date: Fri, 7 May 2021 16:18:28 -0400 Subject: [PATCH 19/19] fix: rename import test --- .../{example1.ipynb => example1_import.ipynb} | 0 tests/test_system_tests.py | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) rename tests/repo-like-with-imports/{example1.ipynb => example1_import.ipynb} (100%) diff --git a/tests/repo-like-with-imports/example1.ipynb b/tests/repo-like-with-imports/example1_import.ipynb similarity index 100% rename from tests/repo-like-with-imports/example1.ipynb rename to tests/repo-like-with-imports/example1_import.ipynb diff --git a/tests/test_system_tests.py b/tests/test_system_tests.py index d6bb26c..2e529af 100644 --- a/tests/test_system_tests.py +++ b/tests/test_system_tests.py @@ -67,9 +67,9 @@ def test_repo2cwl_with_imports(self): 0, repo2cwl(['-o', output_dir, self.repo_like_dir_imports]) ) - self.assertListEqual(['example1.cwl'], [f for f in os.listdir(output_dir) if not f.startswith('.')]) + self.assertListEqual(['example1_import.cwl'], [f for f in os.listdir(output_dir) if not f.startswith('.')]) - with open(os.path.join(output_dir, 'example1.cwl')) as f: + with open(os.path.join(output_dir, 'example1_import.cwl')) as f: print('workflow file') print(20 * '=') print(f.read()) @@ -83,7 +83,7 @@ def test_repo2cwl_with_imports(self): fac = cwltool.factory.Factory(runtime_context=runtime_context) try: - example1_tool = fac.make(os.path.join(output_dir, 'example1.cwl')) + example1_tool = fac.make(os.path.join(output_dir, 'example1_import.cwl')) except WorkflowException: self.fail("Execution failed")