Skip to content

Commit

Permalink
fix #76
Browse files Browse the repository at this point in the history
  • Loading branch information
@skoulouzis
skoulouzis committed Mar 22, 2022
1 parent 317bfbc commit 580eebd
Show file tree
Hide file tree
Showing 4 changed files with 81 additions and 57 deletions.
91 changes: 43 additions & 48 deletions jupyterlab_vre/faircell.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,50 +3,48 @@
import ast

class Cell:

title : str
task_name : str
original_source : str
inputs : list
outputs : list
params : list
confs : dict
dependencies : list
chart_obj : dict
node_id : str
container_source : str
cell_hash : int
global_conf : dict
title: str
task_name: str
original_source: str
inputs: list
outputs: list
params: list
confs: dict
dependencies: list
chart_obj: dict
node_id: str
container_source: str
cell_hash: int
global_conf: dict

def __init__(
self,
title,
task_name,
original_source,
inputs,
outputs,
params,
confs,
dependencies,
container_source,
chart_obj = None,
node_id = '',
self,
title,
task_name,
original_source,
inputs,
outputs,
params,
confs,
dependencies,
container_source,
chart_obj=None,
node_id='',
) -> None:

self.title = title
self.task_name = task_name
self.original_source = original_source
self.inputs = list(inputs)
self.outputs = list(outputs)
self.params = list(params)
self.confs = confs
self.types = dict()
self.dependencies = dependencies
self.chart_obj = chart_obj
self.node_id = node_id
self.container_source = container_source
self.cell_hash = int(hashlib.sha1(original_source.encode('utf-8')).hexdigest(), 16)

self.title = title
self.task_name = task_name
self.original_source = original_source
self.inputs = list(inputs)
self.outputs = list(outputs)
self.params = list(params)
self.confs = confs
self.types = dict()
self.dependencies = dependencies
self.chart_obj = chart_obj
self.node_id = node_id
self.container_source = container_source
self.cell_hash = int(hashlib.sha1(original_source.encode('utf-8')).hexdigest(), 16)

def clean_code(self):

Expand All @@ -55,20 +53,19 @@ def clean_code(self):
self.original_source = ""

for line_i in range(0, len(lines)):

line = lines[line_i]
if line.startswith('import') or \
line.startswith('from') or \
line.startswith('#') or \
line.startswith('param_'):
line.startswith('from') or \
line.startswith('#') or \
line.startswith('param_'):
indices_to_remove.append(line_i)

for ir in sorted(indices_to_remove, reverse=True):
lines.pop(ir)

self.original_source = "\n".join(lines)


def integrate_configuration(self):

lines = self.original_source.splitlines()
Expand All @@ -79,7 +76,6 @@ def integrate_configuration(self):

self.original_source = "\n".join(lines)


def generate_dependencies(self):
resolves = []
for d in self.dependencies:
Expand All @@ -91,9 +87,8 @@ def generate_dependencies(self):
resolves.append(resolve_to)
return resolves


def generate_configuration(self):
resolves = []
for c in self.confs:
resolves.append(self.confs[c])
return resolves
return resolves
16 changes: 10 additions & 6 deletions jupyterlab_vre/handlers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

deprecated_modules = {'pathlib': 'pathlib2'}

################################################################################

Expand Down Expand Up @@ -190,21 +191,24 @@ async def post(self, *args, **kwargs):
for dep in current_cell.dependencies:
if 'module' in dep and dep['module']:
if '.' in dep['module']:
set_deps.add((dep['module'].split('.')[0]))
module_name = dep['module'].split('.')[0]
else:
set_deps.add(dep['module'])
module_name = dep['module']
set_deps.add(module_name)
elif 'name' in dep and dep['name']:
module_name = dep['name']
set_deps.add(dep['name'])
if module_name:
if module_name in deprecated_modules.keys():
module_name = deprecated_modules[module_name]
set_deps.add(module_name)

# set_deps = set([dep['module'].split('.')[0] for dep in current_cell.dependencies])

cell_file_path = os.path.join(cell_path, cell_file_name)
dockerfile_file_path = os.path.join(cell_path, dockerfile_name)
env_file_path = os.path.join(cell_path, env_name)
files_info = {}
files_info[cell_file_name] = cell_file_path
files_info[dockerfile_name] = dockerfile_file_path
files_info[env_name] = env_file_path
files_info = {cell_file_name: cell_file_path, dockerfile_name: dockerfile_file_path, env_name: env_file_path}

template_cell.stream(cell=current_cell, deps=deps, types=current_cell.types, confs=confs).dump(cell_file_path)
template_dockerfile.stream(task_name=current_cell.task_name).dump(dockerfile_file_path)
Expand Down
21 changes: 19 additions & 2 deletions jupyterlab_vre/tests/test_handlers.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,24 @@ def add_git_token(self, git_token):
# response_body = json.loads(to_unicode(response.body))
# self.assertNotEqual(response_body, None)

def test_cellshandler_post(self):
cell_index = 1
# def test_cellshandler_post(self):
# cell_index = 1
# git_token = 'test_token'
# self.add_cell(cell_index=cell_index)
# # self.add_git_token(git_token=git_token)
#
# with mock.patch.object(ExtractorHandler, 'get_secure_cookie') as m:
# m.return_value = 'cookie'
# payload = {'port': 'port', 'type': 'type'}
# response = self.fetch('/cellshandler', method='POST', body=json.dumps(payload))
#
# # delete_all_cells()
# response_body = json.loads(to_unicode(response.body))
# self.assertNotEqual(response_body, None)


def test_cellshandler_post_package(self):
cell_index = 0
git_token = 'test_token'
self.add_cell(cell_index=cell_index)
# self.add_git_token(git_token=git_token)
Expand All @@ -78,3 +94,4 @@ def test_cellshandler_post(self):
self.assertNotEqual(response_body, None)



10 changes: 9 additions & 1 deletion requirements.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,12 @@ pyflakes==2.4.0
colorhash==1.0.4
requests==2.27.1
azure-storage-blob==12.9.0
tinydb==4.5.2
tinydb~=4.6.1
github3.py~=3.0.0
tornado~=6.1
notebook~=6.4.8
PyYAML~=6.0
nbformat~=5.1.3
autopep8~=1.6.0
Jinja2~=3.0.3
setuptools~=57.0.0

0 comments on commit 580eebd

Please sign in to comment.