diff --git a/resen/Resen.py b/resen/Resen.py index e183ae1..b21754e 100644 --- a/resen/Resen.py +++ b/resen/Resen.py @@ -76,8 +76,11 @@ def start_bucket(self,bucket_name): def stop_bucket(self,bucket_name): return self.bucket_manager.stop_bucket(bucket_name) - def start_jupyter(self,bucket_name,local,container,lab=True): - return self.bucket_manager.start_jupyter(bucket_name,local,container,lab=lab) + def start_jupyter(self,bucket_name,local,container): + return self.bucket_manager.start_jupyter(bucket_name,local,container) + + def stop_jupyter(self,bucket_name): + return self.bucket_manager.stop_jupyter(bucket_name) def _get_config_dir(self): appname = 'resen' @@ -187,6 +190,9 @@ def create_bucket(self,bucket_name): params['docker']['port'] = list() params['docker']['storage'] = list() params['docker']['status'] = None + params['docker']['jupyter'] = dict() + params['docker']['jupyter']['token'] = None + params['docker']['jupyter']['port'] = None # now add the new bucket to the self.buckets config and then update the config file self.buckets.append(params) @@ -523,7 +529,7 @@ def execute_command(self,bucket_name,command,detach=True): if bucket['docker']['status'] in ['running']: # then we can start the container and update status - result = self.dockerhelper.execute_command(bucket['docker']['container'],command) + result = self.dockerhelper.execute_command(bucket['docker']['container'],command,detach=detach) status, output = result if (detach and status is None) or (not detach and status==0): return True @@ -535,42 +541,42 @@ def execute_command(self,bucket_name,command,detach=True): print('ERROR: Bucket %s is not running!' % (bucket['bucket']['name'])) return False - def start_jupyter(self,bucket_name,local_port,container_port,lab=True): + def start_jupyter(self,bucket_name,local_port,container_port): if not bucket_name in self.bucket_names: print("ERROR: Bucket with name: %s does not exist!" % bucket_name) return False - if lab: - style = 'lab' - else: - style = 'notebook' - - token = '%048x' % random.randrange(16**48) + ind = self.bucket_names.index(bucket_name) + bucket = self.buckets[ind] + pid = self.get_jupyter_pid(bucket['docker']['container']) - command = "bash -cl 'source activate py36 && jupyter %s --no-browser --ip 0.0.0.0 --port %s --NotebookApp.token=%s --KernelSpecManager.ensure_native_kernel=False'" - command = command % (style, container_port, token) + if not pid is None: + port = bucket['docker']['jupyter']['port'] + token = bucket['docker']['jupyter']['token'] + url = 'http://localhost:%s/?token=%s' % (port,token) + print("Jupyter lab is already running and can be accessed in a browser at: %s" % (url)) + return True + + + token = '%048x' % random.randrange(16**48) + command = "bash -cl 'source activate py36 && jupyter lab --no-browser --ip 0.0.0.0 --port %s --NotebookApp.token=%s --KernelSpecManager.ensure_native_kernel=False'" + command = command % (container_port, token) status = self.execute_command(bucket_name,command,detach=True) if status == False: return False time.sleep(0.1) + # now check that jupyter is running self.update_bucket_statuses() - ind = self.bucket_names.index(bucket_name) - bucket = self.buckets[ind] - result = self.dockerhelper.execute_command(bucket['docker']['container'],'ps -ef',detach=False) - output = result[1].decode('utf-8').split('\n') + pid = self.get_jupyter_pid(bucket['docker']['container']) - pid = None - for line in output: - if 'jupyter' in line and token in line: - parsed_line = [x for x in line.split(' ') if x != ''] - pid = parsed_line[1] - break - if pid is not None: + self.buckets[ind]['docker']['jupyter']['token'] = token + self.buckets[ind]['docker']['jupyter']['port'] = local_port + self.save_config() url = 'http://localhost:%s/?token=%s' % (local_port,token) - print("Jupyter %s can be accessed in a browser at: %s" % (style, url)) + print("Jupyter lab can be accessed in a browser at: %s" % (url)) time.sleep(3) webbrowser.open(url) return True @@ -578,6 +584,58 @@ def start_jupyter(self,bucket_name,local_port,container_port,lab=True): print("ERROR: Failed to start jupyter server!") return False + def stop_jupyter(self,bucket_name): + if not bucket_name in self.bucket_names: + print("ERROR: Bucket with name: %s does not exist!" % bucket_name) + return False + + ind = self.bucket_names.index(bucket_name) + bucket = self.buckets[ind] + if not bucket['docker']['status'] in ['running']: + return True + + pid = self.get_jupyter_pid(bucket['docker']['container']) + if pid is None: + return True + + port = bucket['docker']['jupyter']['port'] + python_cmd = 'from notebook.notebookapp import shutdown_server, list_running_servers; ' + python_cmd += 'svrs = [x for x in list_running_servers() if x[\\\"port\\\"] == %s]; ' % (port) + python_cmd += 'sts = True if len(svrs) == 0 else shutdown_server(svrs[0]); print(sts)' + command = "bash -cl '/home/jovyan/envs/py36/bin/python -c \"%s \"'" % (python_cmd) + status = self.execute_command(bucket_name,command,detach=False) + + self.update_bucket_statuses() + + # now verify it is dead + pid = self.get_jupyter_pid(bucket['docker']['container']) + if not pid is None: + print("ERROR: Failed to stop jupyter lab.") + return False + + self.buckets[ind]['docker']['jupyter']['token'] = None + self.buckets[ind]['docker']['jupyter']['port'] = None + self.save_config() + + return True + + def get_jupyter_pid(self,container): + + result = self.dockerhelper.execute_command(container,'ps -ef',detach=False) + if result == False: + return None + + output = result[1].decode('utf-8').split('\n') + + pid = None + for line in output: + if ('jupyter-lab' in line or 'jupyter lab' in line) and '--no-browser --ip 0.0.0.0' in line: + parsed_line = [x for x in line.split(' ') if x != ''] + pid = parsed_line[1] + break + + return pid + def update_bucket_statuses(self): for i,bucket in enumerate(self.buckets): container_id = bucket['docker']['container'] diff --git a/resen/resencmd.py b/resen/resencmd.py index 2f9b42f..bbb0426 100644 --- a/resen/resencmd.py +++ b/resen/resencmd.py @@ -3,7 +3,7 @@ # # Title: resen # -# Author: asreimer +# Author: resen developer team # Description: The resen tool for working with resen-core locally # which allows for listing available core docker # images, creating resen buckets, starting buckets, @@ -17,6 +17,7 @@ import resen import socket import pathlib +import os version = resen.__version__ @@ -34,71 +35,54 @@ def __init__(self,resen): # if def do_create_bucket(self,args): """Usage: -create_bucket bucket_name : Create a new bucket with name bucket_name. Must start with a letter, <=20 characters, and no spaces.""" - inputs,num_inputs = self.parse_args(args) - if num_inputs != 1: - print("Syntax Error") - return +create_bucket : Create a new bucket by responding to the prompts provided.""" - bucket_name = inputs[0] - # check if bucket_name has spaces in it and is greater than 20 characters - # also bucket name must start with a letter - if ' ' in bucket_name or len(bucket_name) > 20 or not bucket_name[0].isalpha(): - print("Syntax Error. Usage: create_bucket bucket_name") - return + # First, ask user for bucket name + print('Please enter a name for your bucket.') + bucket_name = self.get_valid_name('>>> Enter bucket name: ') # First, ask user about the bucket they want to create # resen-core version? valid_versions = sorted([x['version'] for x in self.program.bucket_manager.valid_cores]) - print('Please choose a version of resen-core. Available versions: %s' % ", ".join(valid_versions)) - msg = '>>> Select a version: ' - docker_image = self.get_valid_input(msg,valid_versions) + print('Please choose a version of resen-core.') + docker_image = self.get_valid_version('>>> Select a version: ',valid_versions) # Figure out a port to use local_port = self.get_port() container_port = local_port - # Ask user about storage locations to mount + # Mounting persistent storage + msg = 'Local directories can be mounted to either /home/jovyan/work or ' + msg += '/home/jovyan/mount/ in a bucket. The /home/jovyan/work location is ' + msg += 'a workspace and /home/jovyan/mount/ is intended for mounting in data. ' + msg += 'You will have rw privileges to everything mounted in work, but can ' + msg += 'specified permissions as either r or rw for directories in mount. Code ' + msg += 'and data created in a bucket can ONLY be accessed outside the bucket or ' + msg += 'after the bucket has been deleted if it is saved in a mounted local directory.' + print(msg) mounts = list() - valid_inputs = ['y','n'] - msg = '>>> Mount storage to /home/jovyan/work? (y/n): ' - answer = self.get_valid_input(msg,valid_inputs) + + # query for mount to work + answer = self.get_yn('>>> Mount storage to /home/jovyan/work? (y/n): ') if answer == 'y': - msg = '>>> Enter local path: ' - local_path = self.get_valid_path(msg) + local_path = self.get_valid_local_path('>>> Enter local path: ') container_path = '/home/jovyan/work' permissions = 'rw' mounts.append([local_path,container_path,permissions]) - # query for more mounts - while True: - valid_inputs = ['y','n'] - msg = '>>> Mount additional storage to /home/jovyan/mount? (y/n): ' - answer = self.get_valid_input(msg,valid_inputs) - if answer == 'n': - break - else: - msg = '>>> Enter local path: ' - local_path = self.get_valid_path(msg) - msg = '>>> Enter bucket path: ' - container_path = self.get_valid_path(msg,base='/home/jovyan/mount') - valid_inputs = ['r','rw'] - msg = '>>> Enter permissions (r/rw): ' - permissions = self.get_valid_input(msg,valid_inputs) - mounts.append([local_path,container_path,permissions]) + # query for mounts to mount + answer = self.get_yn('>>> Mount storage to /home/jovyan/mount? (y/n): ') + while answer == 'y': + local_path = self.get_valid_local_path('>>> Enter local path: ') + container_path = self.get_valid_container_path('>>> Enter bucket path: ','/home/jovyan/mount') + permissions = self.get_permissions('>>> Enter permissions (r/rw): ') + mounts.append([local_path,container_path,permissions]) + answer = self.get_yn('>>> Mount additional storage to /home/jovyan/mount? (y/n): ') # should we start jupyterlab when done creating bucket? - valid_inputs = ['y','n'] msg = '>>> Start bucket and jupyterlab? (y/n): ' - start = self.get_valid_input(msg,valid_inputs) == 'y' - - # Now that we have the bucket creation recipe, let's actually create it. - print("Creating bucket with name: %s" % bucket_name) - status = self.program.create_bucket(bucket_name) - if not status: - print("Failed to create bucket!") - return + start = self.get_yn(msg) == 'y' success = True print("...adding core...") @@ -124,32 +108,32 @@ def do_create_bucket(self,args): return # start jupyterlab print("...starting jupyterlab...") - status = self.program.start_jupyter(bucket_name,local_port,container_port,lab=True) + status = self.program.start_jupyter(bucket_name,local_port,container_port) else: print("Failed to create bucket!") status = self.program.remove_bucket(bucket_name) - def do_start_bucket(self,args): - """Usage: -start_bucket bucket_name : Start bucket named bucket_name.""" - inputs,num_inputs = self.parse_args(args) - if num_inputs != 1: - print("Syntax Error. Usage: start_bucket bucket_name") - return +# def do_start_bucket(self,args): +# """Usage: +# start_bucket bucket_name : Start bucket named bucket_name.""" +# inputs,num_inputs = self.parse_args(args) +# if num_inputs != 1: +# print("Syntax Error. Usage: start_bucket bucket_name") +# return - bucket_name = inputs[0] - status = self.program.start_bucket(bucket_name) +# bucket_name = inputs[0] +# status = self.program.start_bucket(bucket_name) - def do_stop_bucket(self,args): - """Usage: -stop_bucket bucket_name : Stop bucket named bucket_name.""" - inputs,num_inputs = self.parse_args(args) - if num_inputs != 1: - print("Syntax Error. Usage: stop_bucket bucket_name") - return +# def do_stop_bucket(self,args): +# """Usage: +# stop_bucket bucket_name : Stop bucket named bucket_name.""" +# inputs,num_inputs = self.parse_args(args) +# if num_inputs != 1: +# print("Syntax Error. Usage: stop_bucket bucket_name") +# return - bucket_name = inputs[0] - status = self.program.stop_bucket(bucket_name) +# bucket_name = inputs[0] +# status = self.program.stop_bucket(bucket_name) def do_remove_bucket(self,args): """Usage: @@ -190,29 +174,56 @@ def do_status(self,args): def do_start_jupyter(self,args): """Usage: ->>> start_jupyter bucket_name local_port bucket_port\t: Start a jupyter notebook server on port bucket_port available at local_port. ->>> start_jupyter bucket_name local_port bucket_port --lab\t: Start a jupyter lab server on port bucket_port available at local_port. +>>> start_jupyter bucket_name : Start jupyter on bucket bucket_name """ inputs,num_inputs = self.parse_args(args) - lab = False - if num_inputs == 3: + + if num_inputs == 1: pass - elif num_inputs == 4: - if inputs[3][0] == '-': - if inputs[3] == '--lab': - lab = True - else: - print("Syntax Error. See 'help start_jupyter'.") - return else: print("Syntax Error. See 'help start_jupyter'.") return + + # get bucket name from input + bucket_name = inputs[0] + + if not bucket_name in self.program.bucket_manager.bucket_names: + print("ERROR: Bucket with name: %s does not exist!" % bucket_name) + return False + + # get bucket infomrmation (ports and status) + # This stuff may be better suited to exist in some kind of "status query" inside of Resen.py + ind = self.program.bucket_manager.bucket_names.index(bucket_name) + bucket = self.program.bucket_manager.buckets[ind] + # This automatically selects the first port in the list of ports + # TODO: Manage multiple ports assigned to one bucket + ports = bucket['docker']['port'][0] + running_status = bucket['docker']['status'] + + + # if bucket is not running, first start bucket + if running_status != 'running': + status = self.program.start_bucket(bucket_name) + + # check if jupyter server running + + # then start jupyter + status = self.program.start_jupyter(bucket_name,ports[0],ports[1]) + + + def do_stop_jupyter(self,args): + """Usage: +stop_jupyter bucket_name : Stop jupyter on bucket bucket_name.""" + inputs,num_inputs = self.parse_args(args) + if num_inputs != 1: + print("Syntax Error. Usage: stop_bucket bucket_name") + return + bucket_name = inputs[0] - local_port = int(inputs[1]) - bucket_port = int(inputs[2]) + status = self.program.stop_jupyter(bucket_name) + status = self.program.stop_bucket(bucket_name) - status = self.program.start_jupyter(bucket_name,local_port,bucket_port,lab=lab) # def do_add_storage(self,args): # """Usage: @@ -319,34 +330,48 @@ def parse_args(self,args): num_inputs = len(inputs) return inputs,num_inputs - def get_valid_input(self,msg,valid_inputs): - if len(valid_inputs) == 1: - valid_msg = valid_inputs[0] - elif len(valid_inputs) == 2: - valid_msg = " or ".join(valid_inputs) - else: - valid_msg = ", ".join(valid_inputs[:-1]) - valid_msg += ", or %s" % valid_inputs[-1] + # The following functions are highly specialized + + def get_yn(self,msg): + valid_inputs = ['y', 'n'] while True: answer = input(msg) - if not valid_inputs: - return answer if answer in valid_inputs: return answer else: - print("Invalid input. Valid inputs are: %s" % valid_msg) + print("Invalid input. Valid input are {} or {}.".format(valid_inputs[0],valid_inputs[1])) - def get_valid_path(self,msg,base=None): + def get_valid_name(self,msg): + print('Valid names may not contain spaces and must start with a letter and be less than 20 characters long.') while True: - answer = input(msg) - path = pathlib.PurePosixPath(answer) - if not base is None: - if base in [str(x) for x in list(path.parents)]: - return str(path) + name = input(msg) + + # check if bucket_name has spaces in it and is greater than 20 characters + # also bucket name must start with a letter + if ' ' in name: + print("Bucket names may not contain spaces.") + elif len(name) > 20: + print("Bucket names must be less than 20 characters.") + elif not name[0].isalpha(): + print("Bucket names must start with an alphabetic character.") + else: + # check if bucket with that name already exists + # Is the only reason create_bucket fails if the name is already take? May need a more rigerous check + status = self.program.create_bucket(name) + if status: + return name else: - print("Invalid path. Must start with: %s" % base) - continue - return str(path) + print("Cannot use the same name as an existing bucket!") + + def get_valid_version(self,msg,valid_versions): + print('Available versions: {}'.format(", ".join(valid_versions))) + while True: + version = input(msg) + if version in valid_versions: + return version + else: + print("Invalid version. Available versions: {}".format(", ".join(valid_versions))) + def get_port(self): # this is not atomic, so it is possible that another process might snatch up the port @@ -360,6 +385,36 @@ def get_port(self): else: port += 1 + def get_valid_local_path(self,msg): + while True: + path = input(msg) + path = pathlib.PurePosixPath(path) + if os.path.isdir(str(path)): + return str(path) + else: + print('Cannot find local path entered.') + + def get_valid_container_path(self,msg,base): + while True: + path = input(msg) + path = pathlib.PurePosixPath(path) + if base in [str(x) for x in list(path.parents)]: + return str(path) + else: + print("Invalid path. Must start with: {}".format(base)) + + def get_permissions(self,msg): + valid_inputs = ['r', 'rw'] + while True: + answer = input(msg) + if answer in valid_inputs: + return answer + else: + print("Invalid input. Valid input are {} or {}.".format(valid_inputs[0],valid_inputs[1])) + + + + def main():