Merge pull request #645 from binpash/refactor-ast

Refactor all Shell ast related code to its own directory (ideally being mostly self contained)

compiler/annotations_utils/util_cmd_invocations.py

@@ -13,7 +13,7 @@
 # for typing
 from pash_annotations.datatypes.CommandInvocationPrefix import CommandInvocationPrefix
 
-from ir_utils import string_to_argument, redir_stdout_to_file, redir_file_to_stdin, make_command
+from shell_ast.ast_util import  string_to_argument, redir_stdout_to_file, redir_file_to_stdin, make_command
 
 def get_command_invocation_prefix_from_dfg_node(dfg_node):
     return CommandInvocationPrefix(cmd_name = dfg_node.com_name,

compiler/annotations_utils/util_parsing.py

@@ -9,7 +9,7 @@
 from pash_annotations.parser.util_parser import get_json_data
 
 
-from ir_utils import format_arg_chars, string_to_argument, log
+from shell_ast.ast_util import format_arg_chars, string_to_argument
 
 
 def merge_to_single_string_with_space(list_str):

compiler/ast_to_ir.py

@@ -1,13 +1,16 @@
-from ast_util import *
+from shell_ast.ast_util import *
 from ir import *
-from definitions.ast_node import *
-from definitions.ast_node_c import *
+from shell_ast.ast_node import *
+from shell_ast.ast_node_c import *
 from util import *
 from parse import from_ast_objects_to_shell
-from expand import *
+from shell_ast.expand import *
 import subprocess
 import config
 
+## TODO: Separate the ir stuff to the bare minimum and 
+##       try to move this to the shell_ast folder.
+
 ##
 ## Compile AST -> Extended AST with IRs
 ##
@@ -291,35 +294,6 @@ def should_expand_arg_char(arg_char):
 def should_expand_argument(argument):
     return any([should_expand_arg_char(arg_char) for arg_char in argument])
 
-def make_echo_ast(argument, var_file_path):
-    nodes = []
-    ## Source variables if present
-    if(not var_file_path is None):
-        arguments = [string_to_argument("source"), string_to_argument(var_file_path)]
-
-        line_number = 0
-        node = make_kv('Command', [line_number, [], arguments, []])
-        nodes.append(node)
-
-    ## Reset the exit status
-    variable_arg = make_kv('V', ['Normal', "false", 'pash_previous_exit_status', []])
-    arguments = [string_to_argument("exit"), [variable_arg]]
-    exit_node = make_kv('Command', [0, [], arguments, []])
-    node = make_kv('Subshell', [0, exit_node, []])
-    nodes.append(node)
-
-    ## Reset the input arguments
-    variable_arg = make_kv('V', ['Normal', "false", 'pash_input_args', []])
-    arguments = [string_to_argument("set"), string_to_argument("--"), [variable_arg]]
-    set_node = make_kv('Command', [0, [], arguments, []])
-    nodes.append(set_node)
-
-    arguments = [string_to_argument("echo"), string_to_argument("-n"), argument]
-
-    line_number = 0
-    node = make_kv('Command', [line_number, [], arguments, []])
-    nodes.append(node)
-    return nodes
 
 ## TODO: Move this function somewhere more general
 def execute_shell_asts(asts):

compiler/config.py

@@ -6,7 +6,6 @@
 
 from datetime import datetime
 
-from ir_utils import *
 from util import *
 
 ## Global
@@ -175,49 +174,49 @@ def add_common_arguments(parser):
 def pass_common_arguments(pash_arguments):
     arguments = []
     if (pash_arguments.no_optimize):
-        arguments.append(string_to_argument("--no_optimize"))
+        arguments.append("--no_optimize")
     if (pash_arguments.dry_run_compiler):
-        arguments.append(string_to_argument("--dry_run_compiler"))
+        arguments.append("--dry_run_compiler")
     if (pash_arguments.assert_compiler_success):
-        arguments.append(string_to_argument("--assert_compiler_success"))
+        arguments.append("--assert_compiler_success")
     if (pash_arguments.avoid_pash_runtime_completion):
-        arguments.append(string_to_argument("--avoid_pash_runtime_completion"))
+        arguments.append("--avoid_pash_runtime_completion")
     if (pash_arguments.profile_driven):
-        arguments.append(string_to_argument("--profile_driven"))
+        arguments.append("--profile_driven")
     if (pash_arguments.output_time):
-        arguments.append(string_to_argument("--output_time"))
+        arguments.append("--output_time")
     if (pash_arguments.output_optimized):
-        arguments.append(string_to_argument("--output_optimized"))
-    arguments.append(string_to_argument("--graphviz"))
-    arguments.append(string_to_argument(pash_arguments.graphviz))
-    arguments.append(string_to_argument("--graphviz_dir"))
-    arguments.append(string_to_argument(pash_arguments.graphviz_dir))
+        arguments.append("--output_optimized")
+    arguments.append("--graphviz")
+    arguments.append(pash_arguments.graphviz)
+    arguments.append("--graphviz_dir")
+    arguments.append(pash_arguments.graphviz_dir)
     if(not pash_arguments.log_file == ""):
-        arguments.append(string_to_argument("--log_file"))
-        arguments.append(string_to_argument(pash_arguments.log_file))
+        arguments.append("--log_file")
+        arguments.append(pash_arguments.log_file)
     if (pash_arguments.no_eager):
-        arguments.append(string_to_argument("--no_eager"))
+        arguments.append("--no_eager")
     if (pash_arguments.no_daemon):
-        arguments.append(string_to_argument("--no_daemon"))
+        arguments.append("--no_daemon")
     if (pash_arguments.distributed_exec):
-        arguments.append(string_to_argument("--distributed_exec"))
+        arguments.append("--distributed_exec")
     if (pash_arguments.parallel_pipelines):
-        arguments.append(string_to_argument("--parallel_pipelines"))
+        arguments.append("--parallel_pipelines")
     if (pash_arguments.daemon_communicates_through_unix_pipes):
-        arguments.append(string_to_argument("--daemon_communicates_through_unix_pipes"))
-    arguments.append(string_to_argument("--r_split_batch_size"))
-    arguments.append(string_to_argument(str(pash_arguments.r_split_batch_size)))
-    arguments.append(string_to_argument("--debug"))
-    arguments.append(string_to_argument(str(pash_arguments.debug)))
-    arguments.append(string_to_argument("--termination"))
-    arguments.append(string_to_argument(pash_arguments.termination))
-    arguments.append(string_to_argument("--speculation"))
-    arguments.append(string_to_argument(pash_arguments.speculation))
-    arguments.append(string_to_argument("--width"))
-    arguments.append(string_to_argument(str(pash_arguments.width)))
+        arguments.append("--daemon_communicates_through_unix_pipes")
+    arguments.append("--r_split_batch_size")
+    arguments.append(str(pash_arguments.r_split_batch_size))
+    arguments.append("--debug")
+    arguments.append(str(pash_arguments.debug))
+    arguments.append("--termination")
+    arguments.append(pash_arguments.termination)
+    arguments.append("--speculation")
+    arguments.append(pash_arguments.speculation)
+    arguments.append("--width")
+    arguments.append(str(pash_arguments.width))
     if(not pash_arguments.config_path == ""):
-        arguments.append(string_to_argument("--config_path"))
-        arguments.append(string_to_argument(pash_arguments.config_path))
+        arguments.append("--config_path")
+        arguments.append(pash_arguments.config_path)
     return arguments
 
 def init_log_file():

compiler/definitions/ir/arg.py

@@ -1,5 +1,5 @@
 from __future__ import annotations
-from ir_utils import *
+from shell_ast.ast_util import *
 from util import *
 
 class Arg:

compiler/definitions/ir/file_id.py

@@ -1,7 +1,7 @@
 import config
 import os
 
-from ir_utils import *
+from shell_ast.ast_util import *
 from util import *
 import uuid
 

compiler/definitions/ir/nodes/pash_split.py

@@ -3,7 +3,6 @@
 
 from definitions.ir.file_id import *
 from definitions.ir.dfg_node import *
-from ir_utils import string_to_argument
 
 import config
 import os

compiler/definitions/ir/nodes/r_split.py

@@ -8,7 +8,7 @@
 
 from definitions.ir.dfg_node import *
 from definitions.ir.file_id import *
-from ir_utils import string_to_argument
+from shell_ast.ast_util import string_to_argument
 
 class RSplit(DFGNode):
     def __init__(self,

compiler/definitions/ir/nodes/r_unwrap.py

@@ -2,7 +2,6 @@
 from pash_annotations.datatypes.CommandInvocationWithIOVars import CommandInvocationWithIOVars
 
 from definitions.ir.dfg_node import *
-from ir_utils import *
 
 class RUnwrap(DFGNode):
     def __init__(self,

compiler/definitions/ir/nodes/r_wrap.py

@@ -4,7 +4,7 @@
 
 from annotations_utils.util_cmd_invocations import to_arg_from_cmd_inv_with_io_vars_without_streaming_inputs_or_outputs_for_wrapping
 from definitions.ir.dfg_node import *
-from ir_utils import *
+from shell_ast.ast_util import *
 
 class RWrap(DFGNode):
     def __init__(self,

compiler/definitions/ir/redirection.py

@@ -1,5 +1,5 @@
 from definitions.ir.arg import *
-from ir_utils import *
+from shell_ast.ast_util import *
 
 class Redirection():
     def __init__(self, redirection):

compiler/definitions/ir/resource.py

@@ -1,7 +1,8 @@
+import socket
+
 from definitions.ir.arg import *
 from util import *
-from ir_utils import *
-import socket
+from shell_ast.ast_util import *
 
 ## TODO: Resources should probably be more elaborate than just a
 ## string and a line range. They could be URLs, and possibly other things.

compiler/ir.py

@@ -21,7 +21,7 @@
 import definitions.ir.nodes.r_wrap as r_wrap
 import definitions.ir.nodes.r_unwrap as r_unwrap
 
-from ir_utils import *
+from shell_ast.ast_util import *
 from util import *
 
 import config

compiler/ir_to_ast.py

@@ -1,7 +1,8 @@
 import os
 from datetime import datetime
+
 from util import *
-from ir_utils import *
+from shell_ast.ast_util import *
 from parse import from_ast_objects_to_shell
 import config
 

compiler/json_ast.py

@@ -1,6 +1,6 @@
 import json
 import config
-from definitions.ast_node import CustomJSONEncoder
+from shell_ast.ast_node import CustomJSONEncoder
 from subprocess import run, PIPE
 
 from util import *

compiler/parse.py

@@ -3,11 +3,10 @@
 import subprocess
 import sys
 
-from ast_util import *
-from util import *
-from definitions.ast_node import *
+from shell_ast.ast_util import UnparsedScript
+from shell_ast.ast_node import AstNode, ast_node_to_untyped_deep
 
-sys.path.append(os.path.join(config.PASH_TOP, "compiler/parser/ceda"))
+from util import *
 
 import libdash.parser
 import libdash.printer

compiler/pash.py

@@ -3,7 +3,9 @@
 import subprocess
 import argparse
 from datetime import datetime
-import ast_to_ast
+
+from shell_ast import ast_to_ast
+
 from ir import *
 from parse import parse_shell_to_asts_interactive
 from pash_graphviz import maybe_init_graphviz_dir

compiler/preprocessor/preprocessor.py

@@ -3,7 +3,7 @@
 import os
 
 import config
-import ast_to_ast
+from shell_ast import ast_to_ast
 from ir import FileIdGen
 from parse import parse_shell_to_asts, from_ast_objects_to_shell
 from util import *

compiler/definitions/ast_node.py → compiler/shell_ast/ast_node.py

@@ -1,8 +1,7 @@
 from json import JSONEncoder
 
-from definitions.ast_node_c import *
+from shell_ast.ast_node_c import *
 from definitions.no_match_exception import *
-from ir_utils import *
 from util import *
 
 

compiler/ast_to_ast.py → compiler/shell_ast/ast_to_ast.py

@@ -3,7 +3,7 @@
 
 import config
 
-from ast_util import *
+from shell_ast.ast_util import *
 from parse import from_ast_objects_to_shell_file
 
 
@@ -519,7 +519,8 @@ def make_call_to_pash_runtime(ir_filename, sequential_script_file_name,
                  string_to_argument(sequential_script_file_name),
                  string_to_argument(ir_filename)]
     ## Pass a relevant argument to the planner
-    arguments += config.pass_common_arguments(config.pash_args)
+    common_arguments_strings = config.pass_common_arguments(config.pash_args)
+    arguments += [string_to_argument(string) for string in common_arguments_strings]
     runtime_node = make_command(arguments)
 
     ## Restore the arguments to propagate internal changes, e.g., from `shift` outside.