Merge pull request #628 from binpash/remove-pexpect

Remove pexpect and add a deprecation warning

compiler/config.py

@@ -34,12 +34,6 @@
 annotations = []
 pash_args = None
 
-## Contains a bash subprocess that is used for expanding
-bash_mirror = None
-
-## A cache containing variable values since variables are not meant to change while we compile one region
-variable_cache = {}
-
 ## Increase the recursion limit (it seems that the parser/unparser needs it for bigger graphs)
 sys.setrecursionlimit(10000)
 
@@ -83,9 +77,6 @@ def add_common_arguments(parser):
     parser.add_argument("--avoid_pash_runtime_completion",
                         help="avoid the pash_runtime execution completion (only relevant when --debug > 0)",
                         action="store_true")
-    parser.add_argument("--expand_using_bash_mirror",
-                        help="instead of expanding using the internal expansion code, expand using a bash mirror process (slow)",
-                        action="store_true")
     parser.add_argument("--profile_driven",
                         help="(experimental) use profiling information when optimizing",
                         action="store_true")
@@ -171,8 +162,6 @@ def pass_common_arguments(pash_arguments):
         arguments.append(string_to_argument("--assert_compiler_success"))
     if (pash_arguments.avoid_pash_runtime_completion):
         arguments.append(string_to_argument("--avoid_pash_runtime_completion"))
-    if (pash_arguments.expand_using_bash_mirror):
-        arguments.append(string_to_argument("--expand_using_bash_mirror"))
     if (pash_arguments.profile_driven):
         arguments.append(string_to_argument("--profile_driven"))
     if (pash_arguments.output_time):
@@ -223,80 +212,6 @@ def init_log_file():
         with open(pash_args.log_file, "w") as f:
             pass
 
-def wait_bash_mirror(bash_mirror):
-    r = bash_mirror.expect(r'EXPECT\$ ')
-    assert(r == 0)
-    output = bash_mirror.before
-
-    ## I am not sure why, but \r s are added before \n s
-    output = output.replace('\r\n', '\n')
-
-    log("Before the prompt!")
-    log(output)
-    return output
-
-
-def query_expand_variable_bash_mirror(variable):
-    global bash_mirror
-
-    command = f'if [ -z ${{{variable}+foo}} ]; then echo -n "PASH_VAR_UNSET"; else echo -n "${variable}"; fi'
-    data = sync_run_line_command_mirror(command)
-
-    if data == "PASH_VAR_UNSET":
-        return None
-    else:
-        ## This is here because we haven't specified utf encoding when spawning bash mirror
-        # return data.decode('ascii')
-        return data
-
-def query_expand_bash_mirror(string):
-    global bash_mirror
-
-    command = f'echo -n "{string}"'
-    return sync_run_line_command_mirror(command)
-
-def sync_run_line_command_mirror(command):
-    bash_command = f'{command}'
-    log("Executing bash command in mirror:", bash_command)
-
-    bash_mirror.sendline(bash_command)
-
-    data = wait_bash_mirror(bash_mirror)
-    log("mirror done!")
-
-    return data
-
-
-def update_bash_mirror_vars(var_file_path):
-    global bash_mirror
-
-    assert(var_file_path != ""  and not var_file_path is None)
-
-    bash_mirror.sendline(f'PS1="EXPECT\$ "')
-    wait_bash_mirror(bash_mirror)
-    log("PS1 set!")
-
-    ## TODO: There is unnecessary write/read to this var file now.
-    bash_mirror.sendline(f'source {var_file_path}')
-    log("sent source to mirror")
-    wait_bash_mirror(bash_mirror)
-    log("mirror done!")
-
-def add_to_variable_cache(variable_name, value):
-    global variable_cache
-    variable_cache[variable_name] = value
-
-def get_from_variable_cache(variable_name):
-    global variable_cache
-    try:
-        return variable_cache[variable_name]
-    except:
-        return None
-
-def reset_variable_cache():
-    global variable_cache
-
-    variable_cache = {}
 
 def is_array_variable(token):
     return ('a' in token)

compiler/expand.py

@@ -247,13 +247,6 @@ def lookup_variable(var, _lookup_config):
     ##           eval "$cmd"
 
 
-    ## Only check the cache if we are using bash mirror
-    if config.pash_args.expand_using_bash_mirror:
-        ## If we find the value in the cache just use it
-        var_value = config.get_from_variable_cache(var)
-        if var_value is not None:
-            return None, var_value
-
     if(var == '@'):
         argument_values = lookup_variable_inner_core('pash_input_args')
         expanded_var = " ".join(argument_values)
@@ -282,10 +275,6 @@ def lookup_variable(var, _lookup_config):
         ## TODO: We can pull this to expand any string.
         expanded_var = lookup_variable_inner(var)
 
-    ## Add it to the cache to find it next time
-    if config.pash_args.expand_using_bash_mirror:
-        config.add_to_variable_cache(var, expanded_var)
-
     return None, expanded_var
 
 ## Looksup a variable and flattens it if it is an array 
@@ -307,12 +296,9 @@ def lookup_variable_inner_core(varname):
 
 
 def lookup_variable_inner_unsafe(varname):
-    if config.pash_args.expand_using_bash_mirror:
-        return config.query_expand_variable_bash_mirror(varname)
-    else:
-        ## TODO: Is it in there? If we have -u and it is in there.
-        _type, value = config.config['shell_variables'].get(varname, [None, None])
-        return value
+    ## TODO: Is it in there? If we have -u and it is in there.
+    _type, value = config.config['shell_variables'].get(varname, [None, None])
+    return value
 
 ## This function checks if the -u flag is set
 def is_u_set():

compiler/pash.py

@@ -167,7 +167,12 @@ def parse_args():
     parser.add_argument("-x",
                         help="(experimental) prints commands and their arguments as they execute",
                         action="store_true")
-
+    ## Deprecated argument... keeping here just to output the message
+    ## TODO: Do that with a custom argparse Action (KK: I tried and failed)
+    parser.add_argument("--expand_using_bash_mirror",
+                        help="DEPRECATED: instead of expanding using the internal expansion code, expand using a bash mirror process (slow)",
+                        action="store_true")
+
     config.add_common_arguments(parser)
     args = parser.parse_args()
     config.pash_args = args
@@ -186,6 +191,10 @@ def parse_args():
         log(arg_name, arg_val)
     log("-" * 40)
 
+    ## Print the deprecated argument
+    if args.expand_using_bash_mirror:
+        log("WARNING: Option --expand_using_bash_mirror is deprecated and is *ignored*.", level=0)
+
     ## TODO: We might need to have a better default (like $0 of pa.sh)
     shell_name = "pash"
 

compiler/pash_runtime_daemon.py

@@ -1,8 +1,6 @@
 import argparse
-import pexpect
 import signal
 import socket
-import subprocess
 import sys
 import traceback
 from threading import Thread
@@ -62,31 +60,8 @@ def init():
 
     pash_runtime.runtime_config = config.config['distr_planner']
 
-    if config.pash_args.expand_using_bash_mirror:
-        ## Initialize a bash that is used for expanding
-        ##
-        ## TODO: Alternatively, we could set up a communication with the original bash 
-        ## (though this makes it difficult to have concurrent compilations and execution)
-        ## TODO: We actually need to discuss which arch is better.
-        bash_mirror = init_bash_mirror_subprocess()
-
-        ## Is it OK to save it in config?
-        config.bash_mirror = bash_mirror
-
     return args
 
-def init_bash_mirror_subprocess():
-    ## Spawn a bash process to ask it for expansions
-    p = pexpect.spawn('/usr/bin/env', ['bash', '-i'], 
-                      encoding='utf-8',
-                      echo=False)
-    ## If we are in debug mode also log the bash's output
-    if (config.pash_args.debug >= 1):
-        file_to_save_output = ptempfile()
-        log("bash mirror log saved in:", file_to_save_output)
-        fout = open(file_to_save_output, "w")
-        p.logfile = fout
-    return p
 
 def success_response(string):
     return f'OK: {string}\n'
@@ -279,13 +254,6 @@ def compile_and_add(self, compiled_script_file, var_file, input_ir_file):
         variable_reading_start_time = datetime.now()
         # Read any shell variables files if present
         config.read_vars_file(var_file)
-        if config.pash_args.expand_using_bash_mirror:
-            ## Update the bash mirror with the new variables
-            config.update_bash_mirror_vars(var_file)
-            ## TODO: Maybe we also need to update current directory of bash mirror for file-based expansion?
-
-            ## Clean up the variable cache
-            config.reset_variable_cache()
 
         variable_reading_end_time = datetime.now()
         print_time_delta("Variable Loading", variable_reading_start_time, variable_reading_end_time)
@@ -613,14 +581,6 @@ def close(self):
 def shutdown():
     ## There may be races since this is called through the signal handling
     log("PaSh daemon is shutting down...")
-    if config.bash_mirror is not None:
-        ## TODO: Do we need force
-        ret = config.bash_mirror.terminate(force=True)
-
-        ## The mirror was terminated successfully
-        assert(ret)
-
-        config.bash_mirror.wait()
     log("PaSh daemon shut down successfully...")
 
 def main():

scripts/setup-pash.sh

@@ -21,7 +21,6 @@ mkdir -p $PYTHON_PKG_DIR
 echo "Installing python dependencies..."
 
 python3 -m pip install jsonpickle --root $PYTHON_PKG_DIR --ignore-installed #&> $LOG_DIR/pip_install_jsonpickle.log
-python3 -m pip install pexpect --root $PYTHON_PKG_DIR --ignore-installed #&> $LOG_DIR/pip_install_pexpect.log
 python3 -m pip install graphviz --root $PYTHON_PKG_DIR --ignore-installed #&> $LOG_DIR/pip_install_graphviz.log
 python3 -m pip install numpy --root $PYTHON_PKG_DIR --ignore-installed #&> $LOG_DIR/pip_install_numpy.log
 python3 -m pip install matplotlib --root $PYTHON_PKG_DIR --ignore-installed #&> $LOG_DIR/pip_install_matplotlib.log