From ace1908094e01e844cdfa1584b56749b0c7c63e2 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Fri, 30 Dec 2016 18:09:11 -0500 Subject: [PATCH 1/4] shell_parse: some cosmetic clean ups --- base/shell.jl | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/base/shell.jl b/base/shell.jl index eb091b7922de7..e9aebba337d8e 100644 --- a/base/shell.jl +++ b/base/shell.jl @@ -2,9 +2,9 @@ ## shell-like command parsing ## -function shell_parse(raw::AbstractString, interp::Bool) - s = lstrip(raw) - #Strips the end but respects the space when the string endswith "\\ " +function shell_parse(str::AbstractString, interpolate::Bool=true) + s = lstrip(str) + # strips the end but respects the space when the string ends with "\\ " r = RevString(s) i = start(r) c_old = nothing @@ -22,7 +22,7 @@ function shell_parse(raw::AbstractString, interp::Bool) s = s[1:end-i+1] last_parse = 0:-1 - isempty(s) && return interp ? (Expr(:tuple,:()),last_parse) : ([],last_parse) + isempty(s) && return interpolate ? (Expr(:tuple,:()),last_parse) : ([],last_parse) in_single_quotes = false in_double_quotes = false @@ -57,7 +57,7 @@ function shell_parse(raw::AbstractString, interp::Bool) end j = k end - elseif interp && !in_single_quotes && c == '$' + elseif interpolate && !in_single_quotes && c == '$' update_arg(s[i:j-1]); i = k; j = k if done(s,k) error("\$ right before end of command") @@ -103,18 +103,15 @@ function shell_parse(raw::AbstractString, interp::Bool) update_arg(s[i:end]) append_arg() - if !interp - return (args,last_parse) - end + interpolate || return args, last_parse # construct an expression ex = Expr(:tuple) for arg in args push!(ex.args, Expr(:tuple, arg...)) end - (ex,last_parse) + return ex, last_parse end -shell_parse(s::AbstractString) = shell_parse(s,true) function shell_split(s::AbstractString) parsed = shell_parse(s,false)[1] From b3dfd0db53277f9394a3cf57b817c6fafa4129e1 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Fri, 30 Dec 2016 18:10:02 -0500 Subject: [PATCH 2/4] shell_parse usage: escape characters that will soon be special --- base/managers.jl | 4 ++-- test/replcompletions.jl | 2 +- test/spawn.jl | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/base/managers.jl b/base/managers.jl index 505281ca4dc60..c270183c40587 100644 --- a/base/managers.jl +++ b/base/managers.jl @@ -179,10 +179,10 @@ function launch_on_machine(manager::SSHManager, machine, cnt, params, launched, # the default worker timeout tval = haskey(ENV, "JULIA_WORKER_TIMEOUT") ? - `export JULIA_WORKER_TIMEOUT=$(ENV["JULIA_WORKER_TIMEOUT"]);` : `` + `export JULIA_WORKER_TIMEOUT=$(ENV["JULIA_WORKER_TIMEOUT"])\;` : `` # Julia process with passed in command line flag arguments - cmd = `cd $dir && $tval $exename $exeflags` + cmd = `cd $dir '&&' $tval $exename $exeflags` # shell login (-l) with string command (-c) to launch julia process cmd = `sh -l -c $(shell_escape(cmd))` diff --git a/test/replcompletions.jl b/test/replcompletions.jl index 2a4b7855d298f..6335342596c57 100644 --- a/test/replcompletions.jl +++ b/test/replcompletions.jl @@ -289,7 +289,7 @@ for (T, arg) in [(String,"\")\""),(Char, "')'")] @test s[r] == "CompletionFoo.test2" end -s = "(1, CompletionFoo.test2(`)`," +s = "(1, CompletionFoo.test2(`')'`," c, r, res = test_complete(s) @test c[1] == string(first(methods(Main.CompletionFoo.test2, Tuple{Cmd}))) @test length(c) == 1 diff --git a/test/spawn.jl b/test/spawn.jl index 830771b839805..d1a07dd30b7f4 100644 --- a/test/spawn.jl +++ b/test/spawn.jl @@ -32,7 +32,7 @@ end #### Examples used in the manual #### -@test readstring(`$echo hello | sort`) == "hello | sort\n" +@test readstring(`$echo hello \| sort`) == "hello | sort\n" @test readstring(pipeline(`$echo hello`, sortcmd)) == "hello\n" @test length(spawn(pipeline(`$echo hello`, sortcmd)).processes) == 2 From 4c271fc58e35f57be85042a79ce942ade6a271cd Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Fri, 30 Dec 2016 18:27:15 -0500 Subject: [PATCH 3/4] shell_parse: deprecate unescaped chars "#{}()[]<>|&*?~;" in commands --- base/managers.jl | 4 ++-- base/process.jl | 3 ++- base/shell.jl | 22 +++++++++++++++------- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/base/managers.jl b/base/managers.jl index c270183c40587..ad725c5e2b49a 100644 --- a/base/managers.jl +++ b/base/managers.jl @@ -185,7 +185,7 @@ function launch_on_machine(manager::SSHManager, machine, cnt, params, launched, cmd = `cd $dir '&&' $tval $exename $exeflags` # shell login (-l) with string command (-c) to launch julia process - cmd = `sh -l -c $(shell_escape(cmd))` + cmd = `sh -l -c $(shell_escape(cmd, special = ""))` # remote launch with ssh with given ssh flags / host / port information # -T → disable pseudo-terminal allocation @@ -195,7 +195,7 @@ function launch_on_machine(manager::SSHManager, machine, cnt, params, launched, # forwarded connections are causing collisions # -n → Redirects stdin from /dev/null (actually, prevents reading from stdin). # Used when running ssh in the background. - cmd = `ssh -T -a -x -o ClearAllForwardings=yes -n $sshflags $host $(shell_escape(cmd))` + cmd = `ssh -T -a -x -o ClearAllForwardings=yes -n $sshflags $host $(shell_escape(cmd, special = ""))` # launch the remote Julia process diff --git a/base/process.jl b/base/process.jl index d2d0bddd7b8dd..c8a249d858311 100644 --- a/base/process.jl +++ b/base/process.jl @@ -97,7 +97,8 @@ end hash(x::AndCmds, h::UInt) = hash(x.a, hash(x.b, h)) ==(x::AndCmds, y::AndCmds) = x.a == y.a && x.b == y.b -shell_escape(cmd::Cmd) = shell_escape(cmd.exec...) +shell_escape(cmd::Cmd; special::AbstractString=shell_special) = + shell_escape(cmd.exec..., special=special) function show(io::IO, cmd::Cmd) print_env = cmd.env !== nothing diff --git a/base/shell.jl b/base/shell.jl index e9aebba337d8e..8de19ae87d5d7 100644 --- a/base/shell.jl +++ b/base/shell.jl @@ -2,6 +2,8 @@ ## shell-like command parsing ## +const shell_special = "#{}()[]<>|&*?~;" + function shell_parse(str::AbstractString, interpolate::Bool=true) s = lstrip(str) # strips the end but respects the space when the string ends with "\\ " @@ -92,6 +94,8 @@ function shell_parse(str::AbstractString, interpolate::Bool=true) update_arg(s[i:j-1]); i = k c, k = next(s,k) end + elseif !in_single_quotes && !in_double_quotes && c in shell_special + depwarn("special characters \"$shell_special\" should now be quoted in commands", :shell_parse) end j = k end @@ -122,14 +126,14 @@ function shell_split(s::AbstractString) args end -function print_shell_word(io::IO, word::AbstractString) +function print_shell_word(io::IO, word::AbstractString, special::AbstractString = shell_special) if isempty(word) print(io, "''") end has_single = false has_special = false for c in word - if isspace(c) || c=='\\' || c=='\'' || c=='"' || c=='$' + if isspace(c) || c=='\\' || c=='\'' || c=='"' || c=='$' || c in special has_special = true if c == '\'' has_single = true @@ -152,13 +156,17 @@ function print_shell_word(io::IO, word::AbstractString) end end -function print_shell_escaped(io::IO, cmd::AbstractString, args::AbstractString...) - print_shell_word(io, cmd) +function print_shell_escaped( + io::IO, cmd::AbstractString, args::AbstractString...; + special::AbstractString=shell_special + ) + print_shell_word(io, cmd, special) for arg in args print(io, ' ') - print_shell_word(io, arg) + print_shell_word(io, arg, special) end end -print_shell_escaped(io::IO) = nothing +print_shell_escaped(io::IO; special::String=shell_special) = nothing -shell_escape(args::AbstractString...) = sprint(print_shell_escaped, args...) +shell_escape(args::AbstractString...; special::AbstractString=shell_special) = + sprint(io->print_shell_escaped(io, args..., special=special)) From 8c660de6f67200b34fa5bf3a077c385b52302070 Mon Sep 17 00:00:00 2001 From: Stefan Karpinski Date: Mon, 2 Jan 2017 23:36:20 -0500 Subject: [PATCH 4/4] shell_escape: add a docstring --- base/shell.jl | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/base/shell.jl b/base/shell.jl index 8de19ae87d5d7..2120213e3a3e4 100644 --- a/base/shell.jl +++ b/base/shell.jl @@ -168,5 +168,20 @@ function print_shell_escaped( end print_shell_escaped(io::IO; special::String=shell_special) = nothing +""" + shell_escape(args::Union{Cmd,AbstractString...}; special::AbstractString="$shell_special") + +The unexported `shell_escape` function is the inverse of the unexported `shell_split` function: +it takes a string or command object and escapes any special characters in such a way that calling +`shell_split` on it would give back the array of words in the original command. The `special` +keyword argument controls what characters in addition to whitespace, backslashes, quotes and +dollar signs are considered to be special. Examples: + + julia> Base.shell_escape("echo", "this", "&&", "that") + "echo this '&&' that" + + julia> Base.shell_escape("cat", "/foo/bar baz", "&&", "echo", "done", special="") + "cat '/foo/bar baz' && echo done" +""" shell_escape(args::AbstractString...; special::AbstractString=shell_special) = sprint(io->print_shell_escaped(io, args..., special=special))