Extend pash to support variable passing for spec (#666)

* restore variables

* speculative runtime fix

* nit

* mini change

* nits

* handle stdout

compiler/orchestrator_runtime/pash_source_declare_vars.sh

@@ -11,7 +11,9 @@
 
 filter_vars_file()
 {
-    cat "$1" | grep -v "^declare -\([A-Za-z]\|-\)* \(pash\|BASH\|LINENO\|EUID\|GROUPS\)"
+    cat "$1" | grep -v "^declare -\([A-Za-z]\|-\)* \(pash\|BASH\|LINENO\|EUID\|GROUPS\)" 
+    # The extension below is done for the speculative pash
+    # | grep -v "LS_COLORS"
 }
 
 ## TODO: Error handling if the argument is empty?

compiler/orchestrator_runtime/speculative/speculative_runtime.sh

@@ -1,15 +1,20 @@
 #!/bin/bash
 
 
-## TODO: Ask the scheduler to let us know when a command has been committed and what is its exit code.
 ## TODO: Define the client in pash_spec_init_setup (which should be sourced by pash_init_setup)
 
-## TODO: Then we need to extend the scheduler to also support this protocol (unix sockets only) and 
-##       Respond when the command is actually done.
+pash_redir_output echo "$$: (2) Before asking the scheduler for cmd: ${pash_speculative_command_id} exit code..."
 
-export pash_speculative_command_id=$1
+## TODO: Correctly save variables
+## Save the shell variables to a file (necessary for expansion)
+export pash_runtime_shell_variables_file="${PASH_TMP_PREFIX}/variables_$RANDOM$RANDOM$RANDOM"
+source "$RUNTIME_DIR/pash_declare_vars.sh" "$pash_runtime_shell_variables_file"
+pash_redir_output echo "$$: (1) Bash variables saved in: $pash_runtime_shell_variables_file"
+
+## TODO: We want to send the environment to the scheduler.
+##       Once the scheduler determines if there are environment changes, it can then
+##       decide to rerun or not the speculated commands with the new environment.
 
-pash_redir_output echo "$$: (2) Before asking the scheduler for cmd: ${pash_speculative_command_id} exit code..."
 ## Send and receive from daemon
 msg="Wait:${pash_speculative_command_id}"
 daemon_response=$(pash_spec_communicate_scheduler "$msg") # Blocking step, daemon will not send response until it's safe to continue
@@ -18,7 +23,10 @@ daemon_response=$(pash_spec_communicate_scheduler "$msg") # Blocking step, daemo
 if [[ "$daemon_response" == *"OK:"* ]]; then
     # shellcheck disable=SC2206
     response_args=($daemon_response)
+    pash_redir_output echo "$$: (2) Scheduler responded: $daemon_response"
     cmd_exit_code=${response_args[1]}
+    output_variable_file=${response_args[2]}
+    stdout_file=${response_args[3]}
 elif [ -z "$daemon_response" ]; then
     ## Trouble... Daemon crashed, rip
     pash_redir_output echo "$$: ERROR: (2) Scheduler crashed!"
@@ -32,7 +40,13 @@ fi
 pash_redir_output echo "$$: (2) Scheduler returned exit code: ${cmd_exit_code} for cmd with id: ${pash_speculative_command_id}."
 
 
-## TODO: Figure out if this exits properly (prob not)
 pash_runtime_final_status=${cmd_exit_code}
 
+## TODO: Restore the variables (doesn't work currently because variables are printed using `env`)
+pash_redir_output echo "$$: (2) Recovering script variables from: $output_variable_file"
+# source "$RUNTIME_DIR/pash_source_declare_vars.sh" "$output_variable_file"
+
+pash_redir_output echo "$$: (2) Recovering stdout from: $stdout_file"
+cat "${stdout_file}"
+
 ## TODO: Also need to use wrap_vars maybe to `set` properly etc

compiler/pash_runtime.sh

@@ -4,7 +4,8 @@
 ## pash_sequential_script_file: the sequential script. Just running it should work for all tests.
 ## pash_input_ir_file: the file that contains the IR to be compiled 
 
-## TODO: Determine arguments for speculative
+## When called by spec, assumes this variable is set:
+## pash_spec_command_id: the node id for the specific command
 
 
 ##
@@ -86,10 +87,10 @@ if [ "$pash_speculative_flag" -eq 1 ]; then
     ## we just want to ask the scheduler in (3) to let us know when the df_region
     ## has finished executing and what is its exit code.
 
-    ## The first argument is just the command id
-    export pash_speculative_command_id=$1
+    ## TODO: We probably need to make this a local variable so that POSIX tests pass
+    export pash_speculative_command_id=$pash_spec_command_id
 
-    source "$RUNTIME_DIR/speculative/speculative_runtime.sh" "${pash_speculative_command_id}"
+    source "$RUNTIME_DIR/speculative/speculative_runtime.sh"
 
     ## TODO:
     ## 2. Check the flag in pash.py and if it is set, do the speculative transformation.

compiler/shell_ast/ast_to_ast.py

@@ -545,19 +545,15 @@ def make_call_to_pash_runtime(ir_filename, sequential_script_file_name,
     return runtime_node
 
 ## TODO: Make that an actual call to the spec runtime
-def make_call_to_spec_runtime(command_id: str) -> AstNode:
+def make_call_to_spec_runtime(command_id: int) -> AstNode:
+
+    assignments = [["pash_spec_command_id",
+                        string_to_argument(str(command_id))]]
     ## Call the runtime
     arguments = [string_to_argument("source"),
-                 string_to_argument(config.RUNTIME_EXECUTABLE),
-                 string_to_argument(str(command_id))]
+                 string_to_argument(config.RUNTIME_EXECUTABLE)]
     ## Pass all relevant argument to the planner
-    common_arguments_strings = config.pass_common_arguments(config.pash_args)
-    arguments += [string_to_argument(string) for string in common_arguments_strings]
-    runtime_node = make_command(arguments)
-
-    ## Create generic wrapper commands
-    pre_runtime_nodes = make_pre_runtime_nodes()
-    post_runtime_nodes = make_post_runtime_nodes()
-    nodes = pre_runtime_nodes + [runtime_node] + post_runtime_nodes
-    sequence = make_semi_sequence(nodes)
-    return sequence
+    runtime_node = make_command(arguments,
+                                assignments=assignments)
+
+    return runtime_node