From 55966bb319e846beadad6ea800c6ef746a444ce9 Mon Sep 17 00:00:00 2001 From: Konstantinos Kallas Date: Tue, 21 Feb 2023 14:24:12 -0500 Subject: [PATCH 1/5] Start moving things to shell_ast folder --- compiler/ast_to_ir.py | 34 +++--------------------- compiler/ir_utils.py | 10 ++----- compiler/parse.py | 6 ++--- compiler/pash.py | 4 ++- compiler/preprocessor/preprocessor.py | 2 +- compiler/shell_ast/__init__.py | 0 compiler/{ => shell_ast}/ast_to_ast.py | 2 +- compiler/{ => shell_ast}/ast_util.py | 36 ++++++++++++++++++++++++++ 8 files changed, 49 insertions(+), 45 deletions(-) create mode 100644 compiler/shell_ast/__init__.py rename compiler/{ => shell_ast}/ast_to_ast.py (99%) rename compiler/{ => shell_ast}/ast_util.py (58%) diff --git a/compiler/ast_to_ir.py b/compiler/ast_to_ir.py index a6a4cf69f..c7b12f25f 100644 --- a/compiler/ast_to_ir.py +++ b/compiler/ast_to_ir.py @@ -1,4 +1,4 @@ -from ast_util import * +from shell_ast.ast_util import * from ir import * from definitions.ast_node import * from definitions.ast_node_c import * @@ -8,6 +8,9 @@ import subprocess import config +## TODO: Separate the ir stuff to the bare minimum and +## try to move this to the shell_ast folder. + ## ## Compile AST -> Extended AST with IRs ## @@ -291,35 +294,6 @@ def should_expand_arg_char(arg_char): def should_expand_argument(argument): return any([should_expand_arg_char(arg_char) for arg_char in argument]) -def make_echo_ast(argument, var_file_path): - nodes = [] - ## Source variables if present - if(not var_file_path is None): - arguments = [string_to_argument("source"), string_to_argument(var_file_path)] - - line_number = 0 - node = make_kv('Command', [line_number, [], arguments, []]) - nodes.append(node) - - ## Reset the exit status - variable_arg = make_kv('V', ['Normal', "false", 'pash_previous_exit_status', []]) - arguments = [string_to_argument("exit"), [variable_arg]] - exit_node = make_kv('Command', [0, [], arguments, []]) - node = make_kv('Subshell', [0, exit_node, []]) - nodes.append(node) - - ## Reset the input arguments - variable_arg = make_kv('V', ['Normal', "false", 'pash_input_args', []]) - arguments = [string_to_argument("set"), string_to_argument("--"), [variable_arg]] - set_node = make_kv('Command', [0, [], arguments, []]) - nodes.append(set_node) - - arguments = [string_to_argument("echo"), string_to_argument("-n"), argument] - - line_number = 0 - node = make_kv('Command', [line_number, [], arguments, []]) - nodes.append(node) - return nodes ## TODO: Move this function somewhere more general def execute_shell_asts(asts): diff --git a/compiler/ir_utils.py b/compiler/ir_utils.py index 915e152a9..dbd24e8d8 100644 --- a/compiler/ir_utils.py +++ b/compiler/ir_utils.py @@ -143,14 +143,8 @@ def format_expanded_arg_char(arg_char): ## TODO: Make this correct raise ValueError -## These functions check tuple inputs (configuration and streaming ones) -def is_single_input(inputs): - assert(False) - assert(isinstance(inputs, tuple)) - conf_inputs = inputs[0] - streaming_inputs = inputs[1] - return (len(conf_inputs) == 0 - and len(streaming_inputs) == 1) + +## TODO: This seems like it should go to ast_util ## This function gets a key and a value from the ast json format def get_kv(dic): diff --git a/compiler/parse.py b/compiler/parse.py index 2b0bd78be..a701ab6bf 100644 --- a/compiler/parse.py +++ b/compiler/parse.py @@ -3,11 +3,9 @@ import subprocess import sys -from ast_util import * +from shell_ast.ast_util import UnparsedScript from util import * -from definitions.ast_node import * - -sys.path.append(os.path.join(config.PASH_TOP, "compiler/parser/ceda")) +from definitions.ast_node import AstNode, ast_node_to_untyped_deep import libdash.parser import libdash.printer diff --git a/compiler/pash.py b/compiler/pash.py index 56e91c12f..e79c2cedd 100755 --- a/compiler/pash.py +++ b/compiler/pash.py @@ -3,7 +3,9 @@ import subprocess import argparse from datetime import datetime -import ast_to_ast + +from shell_ast import ast_to_ast + from ir import * from parse import parse_shell_to_asts_interactive from pash_graphviz import maybe_init_graphviz_dir diff --git a/compiler/preprocessor/preprocessor.py b/compiler/preprocessor/preprocessor.py index 99242194f..9b47fc563 100644 --- a/compiler/preprocessor/preprocessor.py +++ b/compiler/preprocessor/preprocessor.py @@ -3,7 +3,7 @@ import os import config -import ast_to_ast +from shell_ast import ast_to_ast from ir import FileIdGen from parse import parse_shell_to_asts, from_ast_objects_to_shell from util import * diff --git a/compiler/shell_ast/__init__.py b/compiler/shell_ast/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/compiler/ast_to_ast.py b/compiler/shell_ast/ast_to_ast.py similarity index 99% rename from compiler/ast_to_ast.py rename to compiler/shell_ast/ast_to_ast.py index f0096a21d..72756f509 100644 --- a/compiler/ast_to_ast.py +++ b/compiler/shell_ast/ast_to_ast.py @@ -3,7 +3,7 @@ import config -from ast_util import * +from shell_ast.ast_util import * from parse import from_ast_objects_to_shell_file diff --git a/compiler/ast_util.py b/compiler/shell_ast/ast_util.py similarity index 58% rename from compiler/ast_util.py rename to compiler/shell_ast/ast_util.py index 7d25be089..2a93b6a0f 100644 --- a/compiler/ast_util.py +++ b/compiler/shell_ast/ast_util.py @@ -1,4 +1,6 @@ from definitions.ast_node import * +## TODO: These calls need to be moved here, they don't make sense in ir_utils +from ir_utils import string_to_argument, make_kv ## This class is used by the preprocessor in ast_to_ir class PreprocessedAST: @@ -51,3 +53,37 @@ def ast_match(ast_node, cases, *args): return ast_match_untyped(ast_node, cases, *args) return cases[ast_node.construct.value](*args)(ast_node) + +## +## Make some nodes +## + +def make_echo_ast(argument, var_file_path): + nodes = [] + ## Source variables if present + if(not var_file_path is None): + arguments = [string_to_argument("source"), string_to_argument(var_file_path)] + + line_number = 0 + node = make_kv('Command', [line_number, [], arguments, []]) + nodes.append(node) + + ## Reset the exit status + variable_arg = make_kv('V', ['Normal', "false", 'pash_previous_exit_status', []]) + arguments = [string_to_argument("exit"), [variable_arg]] + exit_node = make_kv('Command', [0, [], arguments, []]) + node = make_kv('Subshell', [0, exit_node, []]) + nodes.append(node) + + ## Reset the input arguments + variable_arg = make_kv('V', ['Normal', "false", 'pash_input_args', []]) + arguments = [string_to_argument("set"), string_to_argument("--"), [variable_arg]] + set_node = make_kv('Command', [0, [], arguments, []]) + nodes.append(set_node) + + arguments = [string_to_argument("echo"), string_to_argument("-n"), argument] + + line_number = 0 + node = make_kv('Command', [line_number, [], arguments, []]) + nodes.append(node) + return nodes \ No newline at end of file From 5e05bef5310b416264dd46384d67e8590d892042 Mon Sep 17 00:00:00 2001 From: Konstantinos Kallas Date: Tue, 21 Feb 2023 14:53:08 -0500 Subject: [PATCH 2/5] move some more files to shell_ast --- compiler/ast_to_ir.py | 2 +- compiler/{ => shell_ast}/expand.py | 3 ++- compiler/test_expansion.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) rename compiler/{ => shell_ast}/expand.py (99%) diff --git a/compiler/ast_to_ir.py b/compiler/ast_to_ir.py index c7b12f25f..ac3d0ea60 100644 --- a/compiler/ast_to_ir.py +++ b/compiler/ast_to_ir.py @@ -4,7 +4,7 @@ from definitions.ast_node_c import * from util import * from parse import from_ast_objects_to_shell -from expand import * +from shell_ast.expand import * import subprocess import config diff --git a/compiler/expand.py b/compiler/shell_ast/expand.py similarity index 99% rename from compiler/expand.py rename to compiler/shell_ast/expand.py index aac8770ad..636fea2c8 100644 --- a/compiler/expand.py +++ b/compiler/shell_ast/expand.py @@ -5,7 +5,8 @@ import ast_to_ir import config -import parse +## Could be useful for debugging +# import parse ################################################################################ # SAFE EXPANSION ANALYSIS diff --git a/compiler/test_expansion.py b/compiler/test_expansion.py index 5895a8c8a..ca33b2a21 100644 --- a/compiler/test_expansion.py +++ b/compiler/test_expansion.py @@ -1,6 +1,6 @@ import parse import config -import expand +from shell_ast import expand import json_ast import copy From dc73bd8de1e4e603a590e0d0ed0b961cf7c17d0c Mon Sep 17 00:00:00 2001 From: Konstantinos Kallas Date: Tue, 21 Feb 2023 14:58:27 -0500 Subject: [PATCH 3/5] remove obsolete code from ir_utils --- compiler/ir_utils.py | 58 +------------------------------------------- 1 file changed, 1 insertion(+), 57 deletions(-) diff --git a/compiler/ir_utils.py b/compiler/ir_utils.py index dbd24e8d8..396e9bf5b 100644 --- a/compiler/ir_utils.py +++ b/compiler/ir_utils.py @@ -2,63 +2,7 @@ from util import * -## -## Separate the option from the non-option command arguments -## - -def option_args(options): - option_args_inds = option_args_indices(options) - args = [option for option, i in option_args_inds] - return args - -def option_args_indices(options): - non_option_indices = [i for _, i in non_option_args_indices(options)] - non_option_indices_set = set(non_option_indices) - - ## Find the option args by finding the complement of the - ## non-option args - formated_options = [format_arg_chars(opt) for opt in options] - option_args = [(option, i) for i, option in enumerate(formated_options) - if not i in non_option_indices_set] - return option_args - -def non_option_args(options): - non_option_args_inds = non_option_args_indices(options) - args = [option for option, i in non_option_args_inds] - return args - -def non_option_args_indices(options): - formated_options = [format_arg_chars(opt) for opt in options] - # log(formated_options) - - ## TODO: This might need to become more general - ## - ## WARNING: Handling `-` as stdin should not be done for all - ## commands but only for those that have the stdin-hyphen option. - args = [(option, i) for i, option in enumerate(formated_options) - if not option.startswith("-") or option == "-"] - return args - -## This function interleaves option arguments (that might contain Nones) -## with the rest of the arguments -## -## Assumption: rest_arguments does not contain Nones -def interleave_args(opt_arguments, rest_arguments): - assert(all([arg for arg in rest_arguments if not arg is None])) - arguments = opt_arguments - for i in range(len(arguments)): - if(arguments[i] is None): - rest_arg = rest_arguments.pop(0) - arguments[i] = rest_arg - arguments += rest_arguments - return arguments - -def get_command_from_definition(command_definition): - if 'command' in command_definition: - return command_definition['command'] - - log('Possible issue with definition file: Missing command in command definition {}'.format(command_definition)) - return '' +## TODO: Move everything to ast_util def format_args(args): formatted_args = [format_arg_chars(arg_chars) for arg_chars in args] From fefe55f3d702e90d580f1a741db9cdc89eeb6ea1 Mon Sep 17 00:00:00 2001 From: Konstantinos Kallas Date: Tue, 21 Feb 2023 17:51:25 -0500 Subject: [PATCH 4/5] remove ir_utils --- .../annotations_utils/util_cmd_invocations.py | 2 +- compiler/annotations_utils/util_parsing.py | 2 +- compiler/config.py | 61 +++--- compiler/definitions/ast_node.py | 2 +- compiler/definitions/ir/arg.py | 2 +- compiler/definitions/ir/file_id.py | 2 +- compiler/definitions/ir/nodes/pash_split.py | 1 - compiler/definitions/ir/nodes/r_split.py | 2 +- compiler/definitions/ir/nodes/r_unwrap.py | 1 - compiler/definitions/ir/nodes/r_wrap.py | 2 +- compiler/definitions/ir/redirection.py | 2 +- compiler/definitions/ir/resource.py | 5 +- compiler/ir.py | 2 +- compiler/ir_to_ast.py | 3 +- compiler/ir_utils.py | 181 ----------------- compiler/shell_ast/ast_to_ast.py | 3 +- compiler/shell_ast/ast_util.py | 182 +++++++++++++++++- 17 files changed, 226 insertions(+), 229 deletions(-) delete mode 100644 compiler/ir_utils.py diff --git a/compiler/annotations_utils/util_cmd_invocations.py b/compiler/annotations_utils/util_cmd_invocations.py index 8ee2907df..f012b0dd8 100644 --- a/compiler/annotations_utils/util_cmd_invocations.py +++ b/compiler/annotations_utils/util_cmd_invocations.py @@ -13,7 +13,7 @@ # for typing from pash_annotations.datatypes.CommandInvocationPrefix import CommandInvocationPrefix -from ir_utils import string_to_argument, redir_stdout_to_file, redir_file_to_stdin, make_command +from shell_ast.ast_util import string_to_argument, redir_stdout_to_file, redir_file_to_stdin, make_command def get_command_invocation_prefix_from_dfg_node(dfg_node): return CommandInvocationPrefix(cmd_name = dfg_node.com_name, diff --git a/compiler/annotations_utils/util_parsing.py b/compiler/annotations_utils/util_parsing.py index ebeab7853..f4655b9fa 100644 --- a/compiler/annotations_utils/util_parsing.py +++ b/compiler/annotations_utils/util_parsing.py @@ -9,7 +9,7 @@ from pash_annotations.parser.util_parser import get_json_data -from ir_utils import format_arg_chars, string_to_argument, log +from shell_ast.ast_util import format_arg_chars, string_to_argument def merge_to_single_string_with_space(list_str): diff --git a/compiler/config.py b/compiler/config.py index ee1812b07..040b6d4c0 100644 --- a/compiler/config.py +++ b/compiler/config.py @@ -6,7 +6,6 @@ from datetime import datetime -from ir_utils import * from util import * ## Global @@ -175,49 +174,49 @@ def add_common_arguments(parser): def pass_common_arguments(pash_arguments): arguments = [] if (pash_arguments.no_optimize): - arguments.append(string_to_argument("--no_optimize")) + arguments.append("--no_optimize") if (pash_arguments.dry_run_compiler): - arguments.append(string_to_argument("--dry_run_compiler")) + arguments.append("--dry_run_compiler") if (pash_arguments.assert_compiler_success): - arguments.append(string_to_argument("--assert_compiler_success")) + arguments.append("--assert_compiler_success") if (pash_arguments.avoid_pash_runtime_completion): - arguments.append(string_to_argument("--avoid_pash_runtime_completion")) + arguments.append("--avoid_pash_runtime_completion") if (pash_arguments.profile_driven): - arguments.append(string_to_argument("--profile_driven")) + arguments.append("--profile_driven") if (pash_arguments.output_time): - arguments.append(string_to_argument("--output_time")) + arguments.append("--output_time") if (pash_arguments.output_optimized): - arguments.append(string_to_argument("--output_optimized")) - arguments.append(string_to_argument("--graphviz")) - arguments.append(string_to_argument(pash_arguments.graphviz)) - arguments.append(string_to_argument("--graphviz_dir")) - arguments.append(string_to_argument(pash_arguments.graphviz_dir)) + arguments.append("--output_optimized") + arguments.append("--graphviz") + arguments.append(pash_arguments.graphviz) + arguments.append("--graphviz_dir") + arguments.append(pash_arguments.graphviz_dir) if(not pash_arguments.log_file == ""): - arguments.append(string_to_argument("--log_file")) - arguments.append(string_to_argument(pash_arguments.log_file)) + arguments.append("--log_file") + arguments.append(pash_arguments.log_file) if (pash_arguments.no_eager): - arguments.append(string_to_argument("--no_eager")) + arguments.append("--no_eager") if (pash_arguments.no_daemon): - arguments.append(string_to_argument("--no_daemon")) + arguments.append("--no_daemon") if (pash_arguments.distributed_exec): - arguments.append(string_to_argument("--distributed_exec")) + arguments.append("--distributed_exec") if (pash_arguments.parallel_pipelines): - arguments.append(string_to_argument("--parallel_pipelines")) + arguments.append("--parallel_pipelines") if (pash_arguments.daemon_communicates_through_unix_pipes): - arguments.append(string_to_argument("--daemon_communicates_through_unix_pipes")) - arguments.append(string_to_argument("--r_split_batch_size")) - arguments.append(string_to_argument(str(pash_arguments.r_split_batch_size))) - arguments.append(string_to_argument("--debug")) - arguments.append(string_to_argument(str(pash_arguments.debug))) - arguments.append(string_to_argument("--termination")) - arguments.append(string_to_argument(pash_arguments.termination)) - arguments.append(string_to_argument("--speculation")) - arguments.append(string_to_argument(pash_arguments.speculation)) - arguments.append(string_to_argument("--width")) - arguments.append(string_to_argument(str(pash_arguments.width))) + arguments.append("--daemon_communicates_through_unix_pipes") + arguments.append("--r_split_batch_size") + arguments.append(str(pash_arguments.r_split_batch_size)) + arguments.append("--debug") + arguments.append(str(pash_arguments.debug)) + arguments.append("--termination") + arguments.append(pash_arguments.termination) + arguments.append("--speculation") + arguments.append(pash_arguments.speculation) + arguments.append("--width") + arguments.append(str(pash_arguments.width)) if(not pash_arguments.config_path == ""): - arguments.append(string_to_argument("--config_path")) - arguments.append(string_to_argument(pash_arguments.config_path)) + arguments.append("--config_path") + arguments.append(pash_arguments.config_path) return arguments def init_log_file(): diff --git a/compiler/definitions/ast_node.py b/compiler/definitions/ast_node.py index c1731ca1a..ee1539008 100644 --- a/compiler/definitions/ast_node.py +++ b/compiler/definitions/ast_node.py @@ -2,7 +2,7 @@ from definitions.ast_node_c import * from definitions.no_match_exception import * -from ir_utils import * +from shell_ast.ast_util import * from util import * diff --git a/compiler/definitions/ir/arg.py b/compiler/definitions/ir/arg.py index 8ca591733..b75627862 100644 --- a/compiler/definitions/ir/arg.py +++ b/compiler/definitions/ir/arg.py @@ -1,5 +1,5 @@ from __future__ import annotations -from ir_utils import * +from shell_ast.ast_util import * from util import * class Arg: diff --git a/compiler/definitions/ir/file_id.py b/compiler/definitions/ir/file_id.py index 215d45c49..ecee07ec0 100644 --- a/compiler/definitions/ir/file_id.py +++ b/compiler/definitions/ir/file_id.py @@ -1,7 +1,7 @@ import config import os -from ir_utils import * +from shell_ast.ast_util import * from util import * import uuid diff --git a/compiler/definitions/ir/nodes/pash_split.py b/compiler/definitions/ir/nodes/pash_split.py index add232947..621334807 100644 --- a/compiler/definitions/ir/nodes/pash_split.py +++ b/compiler/definitions/ir/nodes/pash_split.py @@ -3,7 +3,6 @@ from definitions.ir.file_id import * from definitions.ir.dfg_node import * -from ir_utils import string_to_argument import config import os diff --git a/compiler/definitions/ir/nodes/r_split.py b/compiler/definitions/ir/nodes/r_split.py index 3f4b5bc3f..ac2462b49 100644 --- a/compiler/definitions/ir/nodes/r_split.py +++ b/compiler/definitions/ir/nodes/r_split.py @@ -8,7 +8,7 @@ from definitions.ir.dfg_node import * from definitions.ir.file_id import * -from ir_utils import string_to_argument +from shell_ast.ast_util import string_to_argument class RSplit(DFGNode): def __init__(self, diff --git a/compiler/definitions/ir/nodes/r_unwrap.py b/compiler/definitions/ir/nodes/r_unwrap.py index 112b72148..931507220 100644 --- a/compiler/definitions/ir/nodes/r_unwrap.py +++ b/compiler/definitions/ir/nodes/r_unwrap.py @@ -2,7 +2,6 @@ from pash_annotations.datatypes.CommandInvocationWithIOVars import CommandInvocationWithIOVars from definitions.ir.dfg_node import * -from ir_utils import * class RUnwrap(DFGNode): def __init__(self, diff --git a/compiler/definitions/ir/nodes/r_wrap.py b/compiler/definitions/ir/nodes/r_wrap.py index 7149a8cec..1d045a838 100644 --- a/compiler/definitions/ir/nodes/r_wrap.py +++ b/compiler/definitions/ir/nodes/r_wrap.py @@ -4,7 +4,7 @@ from annotations_utils.util_cmd_invocations import to_arg_from_cmd_inv_with_io_vars_without_streaming_inputs_or_outputs_for_wrapping from definitions.ir.dfg_node import * -from ir_utils import * +from shell_ast.ast_util import * class RWrap(DFGNode): def __init__(self, diff --git a/compiler/definitions/ir/redirection.py b/compiler/definitions/ir/redirection.py index 7be6c496a..2203a5a6e 100644 --- a/compiler/definitions/ir/redirection.py +++ b/compiler/definitions/ir/redirection.py @@ -1,5 +1,5 @@ from definitions.ir.arg import * -from ir_utils import * +from shell_ast.ast_util import * class Redirection(): def __init__(self, redirection): diff --git a/compiler/definitions/ir/resource.py b/compiler/definitions/ir/resource.py index 999792cd9..c6ad69c5e 100644 --- a/compiler/definitions/ir/resource.py +++ b/compiler/definitions/ir/resource.py @@ -1,7 +1,8 @@ +import socket + from definitions.ir.arg import * from util import * -from ir_utils import * -import socket +from shell_ast.ast_util import * ## TODO: Resources should probably be more elaborate than just a ## string and a line range. They could be URLs, and possibly other things. diff --git a/compiler/ir.py b/compiler/ir.py index 4f42ab49b..5d9de8675 100644 --- a/compiler/ir.py +++ b/compiler/ir.py @@ -21,7 +21,7 @@ import definitions.ir.nodes.r_wrap as r_wrap import definitions.ir.nodes.r_unwrap as r_unwrap -from ir_utils import * +from shell_ast.ast_util import * from util import * import config diff --git a/compiler/ir_to_ast.py b/compiler/ir_to_ast.py index 078466669..ed75852b2 100644 --- a/compiler/ir_to_ast.py +++ b/compiler/ir_to_ast.py @@ -1,7 +1,8 @@ import os from datetime import datetime + from util import * -from ir_utils import * +from shell_ast.ast_util import * from parse import from_ast_objects_to_shell import config diff --git a/compiler/ir_utils.py b/compiler/ir_utils.py deleted file mode 100644 index 396e9bf5b..000000000 --- a/compiler/ir_utils.py +++ /dev/null @@ -1,181 +0,0 @@ -### Utils - -from util import * - -## TODO: Move everything to ast_util - -def format_args(args): - formatted_args = [format_arg_chars(arg_chars) for arg_chars in args] - return formatted_args - -def format_arg_chars(arg_chars): - chars = [format_arg_char(arg_char) for arg_char in arg_chars] - return "".join(chars) - -## -## BIG TODO: Fix the formating of arg_chars bask to shell scripts and string. -## We need to do this the proper way using the parser. -## -def format_arg_char(arg_char): - key, val = get_kv(arg_char) - if (key == 'C'): - return str(chr(val)) - elif (key == 'B'): - # The $() is just for illustration. This is backticks - return '$({})'.format(val) - elif (key == 'Q'): - formated_val = format_arg_chars(val) - return '"{}"'.format(formated_val) - elif (key == 'V'): - return '${{{}}}'.format(val[2]) - elif (key == 'E'): - ## TODO: This is not right. I think the main reason for the - ## problems is the differences between bash and the posix - ## standard. - # log(" -- escape-debug -- ", val, chr(val)) - non_escape_chars = [92, # \ - 61, # = - 91, # [ - 93, # ] - 45, # - - 58, # : - 126,# ~ - 42] # * - if(val in non_escape_chars): - return '{}'.format(chr(val)) - else: - return '\{}'.format(chr(val)) - else: - log("Cannot format arg_char:", arg_char) - ## TODO: Make this correct - raise NotImplementedError - -## This function finds the first raw character in an argument. -## It needs to be called on an expanded string. -def format_expanded_arg_chars(arg_chars): - chars = [format_expanded_arg_char(arg_char) for arg_char in arg_chars] - return "".join(chars) - -def format_expanded_arg_char(arg_char): - key, val = get_kv(arg_char) - if (key == 'C'): - return str(chr(val)) - elif (key == 'Q'): - formated_val = format_expanded_arg_chars(val) - return '{}'.format(formated_val) - elif (key == 'E'): - ## TODO: I am not sure if this should add \ or not - ## - ## TODO: This is not right. I think the main reason for the - ## problems is the differences between bash and the posix - ## standard. - # log(" -- escape-debug -- ", val, chr(val)) - non_escape_chars = [92, # \ - 61, # = - 91, # [ - 93, # ] - 45, # - - 58, # : - 126,# ~ - 42] # * - if(val in non_escape_chars): - return '{}'.format(chr(val)) - else: - return '\{}'.format(chr(val)) - else: - log("Expanded arg char should not contain:", arg_char) - ## TODO: Make this correct - raise ValueError - - -## TODO: This seems like it should go to ast_util - -## This function gets a key and a value from the ast json format -def get_kv(dic): - return (dic[0], dic[1]) - -def make_kv(key, val): - return [key, val] - -def string_to_arguments(string): - return [string_to_argument(word) for word in string.split(" ")] - -def string_to_argument(string): - ret = [char_to_arg_char(char) for char in string] - return ret - -## FIXME: This is certainly not complete. It is used to generate the -## AST for the call to the distributed planner. It only handles simple -## characters -def char_to_arg_char(char): - return ['C' , ord(char)] - -def escaped_char(char): - return ['E' , ord(char)] - -def standard_var_ast(string): - return make_kv("V", ["Normal", False, string, []]) - -def make_quoted_variable(string): - return make_kv("Q", [standard_var_ast(string)]) - -def quote_arg(arg): - return make_kv("Q", arg) - -def redir_append_stderr_to_string_file(string): - return make_kv("File",["Append",2,string_to_argument(string)]) - -def redir_stdout_to_file(arg): - return make_kv("File",["To", 1, arg]) - -def redir_file_to_stdin(arg): - return make_kv("File",["From", 0, arg]) - -def make_background(body, redirections=[]): - lineno = 0 - node = make_kv("Background", [lineno, body, redirections]) - return node - -def make_backquote(node): - node = make_kv("B", node) - return node - -def make_subshell(body, redirections=[]): - lineno = 0 - node = make_kv("Subshell", [lineno, body, redirections]) - return node - -def make_command(arguments, redirections=[], assignments=[]): - lineno = 0 - node = make_kv("Command", [lineno, assignments, arguments, redirections]) - return node - -def make_nop(): - return make_command([string_to_argument(":")]) - -def make_assignment(var, value): - lineno = 0 - assignment=(var, value) - assignments=[assignment] - node = make_kv("Command", [lineno, assignments, [], []]) - return node - -def make_semi_sequence(asts): - if(len(asts) == 0): - return make_nop() - - if(len(asts) == 1): - return asts[0] - else: - acc = asts[-1] - ## Remove the last ast - iter_asts = asts[:-1] - for ast in iter_asts[::-1]: - acc = make_kv("Semi", [ast, acc]) - return acc - -def make_defun(name, body): - lineno = 0 - node = make_kv("Defun", [lineno, name, body]) - return node - diff --git a/compiler/shell_ast/ast_to_ast.py b/compiler/shell_ast/ast_to_ast.py index 72756f509..4d99064c5 100644 --- a/compiler/shell_ast/ast_to_ast.py +++ b/compiler/shell_ast/ast_to_ast.py @@ -519,7 +519,8 @@ def make_call_to_pash_runtime(ir_filename, sequential_script_file_name, string_to_argument(sequential_script_file_name), string_to_argument(ir_filename)] ## Pass a relevant argument to the planner - arguments += config.pass_common_arguments(config.pash_args) + common_arguments_strings = config.pass_common_arguments(config.pash_args) + arguments += [string_to_argument(string) for string in common_arguments_strings] runtime_node = make_command(arguments) ## Restore the arguments to propagate internal changes, e.g., from `shift` outside. diff --git a/compiler/shell_ast/ast_util.py b/compiler/shell_ast/ast_util.py index 2a93b6a0f..19bd594a9 100644 --- a/compiler/shell_ast/ast_util.py +++ b/compiler/shell_ast/ast_util.py @@ -1,6 +1,7 @@ + from definitions.ast_node import * -## TODO: These calls need to be moved here, they don't make sense in ir_utils -from ir_utils import string_to_argument, make_kv +from util import * + ## This class is used by the preprocessor in ast_to_ir class PreprocessedAST: @@ -54,6 +55,183 @@ def ast_match(ast_node, cases, *args): return cases[ast_node.construct.value](*args)(ast_node) + + +def format_args(args): + formatted_args = [format_arg_chars(arg_chars) for arg_chars in args] + return formatted_args + +def format_arg_chars(arg_chars): + chars = [format_arg_char(arg_char) for arg_char in arg_chars] + return "".join(chars) + +## +## BIG TODO: Fix the formating of arg_chars bask to shell scripts and string. +## We need to do this the proper way using the parser. +## +def format_arg_char(arg_char): + key, val = get_kv(arg_char) + if (key == 'C'): + return str(chr(val)) + elif (key == 'B'): + # The $() is just for illustration. This is backticks + return '$({})'.format(val) + elif (key == 'Q'): + formated_val = format_arg_chars(val) + return '"{}"'.format(formated_val) + elif (key == 'V'): + return '${{{}}}'.format(val[2]) + elif (key == 'E'): + ## TODO: This is not right. I think the main reason for the + ## problems is the differences between bash and the posix + ## standard. + # log(" -- escape-debug -- ", val, chr(val)) + non_escape_chars = [92, # \ + 61, # = + 91, # [ + 93, # ] + 45, # - + 58, # : + 126,# ~ + 42] # * + if(val in non_escape_chars): + return '{}'.format(chr(val)) + else: + return '\{}'.format(chr(val)) + else: + log("Cannot format arg_char:", arg_char) + ## TODO: Make this correct + raise NotImplementedError + +## This function finds the first raw character in an argument. +## It needs to be called on an expanded string. +def format_expanded_arg_chars(arg_chars): + chars = [format_expanded_arg_char(arg_char) for arg_char in arg_chars] + return "".join(chars) + +def format_expanded_arg_char(arg_char): + key, val = get_kv(arg_char) + if (key == 'C'): + return str(chr(val)) + elif (key == 'Q'): + formated_val = format_expanded_arg_chars(val) + return '{}'.format(formated_val) + elif (key == 'E'): + ## TODO: I am not sure if this should add \ or not + ## + ## TODO: This is not right. I think the main reason for the + ## problems is the differences between bash and the posix + ## standard. + # log(" -- escape-debug -- ", val, chr(val)) + non_escape_chars = [92, # \ + 61, # = + 91, # [ + 93, # ] + 45, # - + 58, # : + 126,# ~ + 42] # * + if(val in non_escape_chars): + return '{}'.format(chr(val)) + else: + return '\{}'.format(chr(val)) + else: + log("Expanded arg char should not contain:", arg_char) + ## TODO: Make this correct + raise ValueError + + +## TODO: This seems like it should go to ast_util + +## This function gets a key and a value from the ast json format +def get_kv(dic): + return (dic[0], dic[1]) + +def make_kv(key, val): + return [key, val] + +def string_to_arguments(string): + return [string_to_argument(word) for word in string.split(" ")] + +def string_to_argument(string): + ret = [char_to_arg_char(char) for char in string] + return ret + +## FIXME: This is certainly not complete. It is used to generate the +## AST for the call to the distributed planner. It only handles simple +## characters +def char_to_arg_char(char): + return ['C' , ord(char)] + +def escaped_char(char): + return ['E' , ord(char)] + +def standard_var_ast(string): + return make_kv("V", ["Normal", False, string, []]) + +def make_quoted_variable(string): + return make_kv("Q", [standard_var_ast(string)]) + +def quote_arg(arg): + return make_kv("Q", arg) + +def redir_append_stderr_to_string_file(string): + return make_kv("File",["Append",2,string_to_argument(string)]) + +def redir_stdout_to_file(arg): + return make_kv("File",["To", 1, arg]) + +def redir_file_to_stdin(arg): + return make_kv("File",["From", 0, arg]) + +def make_background(body, redirections=[]): + lineno = 0 + node = make_kv("Background", [lineno, body, redirections]) + return node + +def make_backquote(node): + node = make_kv("B", node) + return node + +def make_subshell(body, redirections=[]): + lineno = 0 + node = make_kv("Subshell", [lineno, body, redirections]) + return node + +def make_command(arguments, redirections=[], assignments=[]): + lineno = 0 + node = make_kv("Command", [lineno, assignments, arguments, redirections]) + return node + +def make_nop(): + return make_command([string_to_argument(":")]) + +def make_assignment(var, value): + lineno = 0 + assignment=(var, value) + assignments=[assignment] + node = make_kv("Command", [lineno, assignments, [], []]) + return node + +def make_semi_sequence(asts): + if(len(asts) == 0): + return make_nop() + + if(len(asts) == 1): + return asts[0] + else: + acc = asts[-1] + ## Remove the last ast + iter_asts = asts[:-1] + for ast in iter_asts[::-1]: + acc = make_kv("Semi", [ast, acc]) + return acc + +def make_defun(name, body): + lineno = 0 + node = make_kv("Defun", [lineno, name, body]) + return node + ## ## Make some nodes ## From 6e21f2ba7c1b747e0df8a05d948d8d271b425577 Mon Sep 17 00:00:00 2001 From: Konstantinos Kallas Date: Tue, 21 Feb 2023 18:04:32 -0500 Subject: [PATCH 5/5] Move make_kv, get_kv to util --- compiler/ast_to_ir.py | 4 ++-- compiler/json_ast.py | 2 +- compiler/parse.py | 3 ++- compiler/{definitions => shell_ast}/ast_node.py | 3 +-- compiler/{definitions => shell_ast}/ast_node_c.py | 0 compiler/shell_ast/ast_util.py | 10 +--------- compiler/shell_ast/expand.py | 4 ++-- compiler/util.py | 6 ++++++ 8 files changed, 15 insertions(+), 17 deletions(-) rename compiler/{definitions => shell_ast}/ast_node.py (99%) rename compiler/{definitions => shell_ast}/ast_node_c.py (100%) diff --git a/compiler/ast_to_ir.py b/compiler/ast_to_ir.py index ac3d0ea60..b5ce38624 100644 --- a/compiler/ast_to_ir.py +++ b/compiler/ast_to_ir.py @@ -1,7 +1,7 @@ from shell_ast.ast_util import * from ir import * -from definitions.ast_node import * -from definitions.ast_node_c import * +from shell_ast.ast_node import * +from shell_ast.ast_node_c import * from util import * from parse import from_ast_objects_to_shell from shell_ast.expand import * diff --git a/compiler/json_ast.py b/compiler/json_ast.py index 89583fea9..0342b5797 100644 --- a/compiler/json_ast.py +++ b/compiler/json_ast.py @@ -1,6 +1,6 @@ import json import config -from definitions.ast_node import CustomJSONEncoder +from shell_ast.ast_node import CustomJSONEncoder from subprocess import run, PIPE from util import * diff --git a/compiler/parse.py b/compiler/parse.py index a701ab6bf..652f7c4e5 100644 --- a/compiler/parse.py +++ b/compiler/parse.py @@ -4,8 +4,9 @@ import sys from shell_ast.ast_util import UnparsedScript +from shell_ast.ast_node import AstNode, ast_node_to_untyped_deep + from util import * -from definitions.ast_node import AstNode, ast_node_to_untyped_deep import libdash.parser import libdash.printer diff --git a/compiler/definitions/ast_node.py b/compiler/shell_ast/ast_node.py similarity index 99% rename from compiler/definitions/ast_node.py rename to compiler/shell_ast/ast_node.py index ee1539008..e83fbe3d6 100644 --- a/compiler/definitions/ast_node.py +++ b/compiler/shell_ast/ast_node.py @@ -1,8 +1,7 @@ from json import JSONEncoder -from definitions.ast_node_c import * +from shell_ast.ast_node_c import * from definitions.no_match_exception import * -from shell_ast.ast_util import * from util import * diff --git a/compiler/definitions/ast_node_c.py b/compiler/shell_ast/ast_node_c.py similarity index 100% rename from compiler/definitions/ast_node_c.py rename to compiler/shell_ast/ast_node_c.py diff --git a/compiler/shell_ast/ast_util.py b/compiler/shell_ast/ast_util.py index 19bd594a9..21c588a19 100644 --- a/compiler/shell_ast/ast_util.py +++ b/compiler/shell_ast/ast_util.py @@ -1,5 +1,5 @@ -from definitions.ast_node import * +from shell_ast.ast_node import * from util import * @@ -141,14 +141,6 @@ def format_expanded_arg_char(arg_char): raise ValueError -## TODO: This seems like it should go to ast_util - -## This function gets a key and a value from the ast json format -def get_kv(dic): - return (dic[0], dic[1]) - -def make_kv(key, val): - return [key, val] def string_to_arguments(string): return [string_to_argument(word) for word in string.split(" ")] diff --git a/compiler/shell_ast/expand.py b/compiler/shell_ast/expand.py index 636fea2c8..b1f231691 100644 --- a/compiler/shell_ast/expand.py +++ b/compiler/shell_ast/expand.py @@ -1,7 +1,7 @@ import copy -from definitions.ast_node import * -from definitions.ast_node_c import * +from shell_ast.ast_node import * +from shell_ast.ast_node_c import * import ast_to_ir import config diff --git a/compiler/util.py b/compiler/util.py index 04ca99fba..a09a24da7 100644 --- a/compiler/util.py +++ b/compiler/util.py @@ -75,3 +75,9 @@ def return_default_if_none_else_itself(arg: Optional[TType], default: TType) -> else: return arg +## This function gets a key and a value from the ast json format +def get_kv(dic): + return (dic[0], dic[1]) + +def make_kv(key, val): + return [key, val]