From 076675828e4a0cdc5e6a7bbdefbbd98f3a0c8810 Mon Sep 17 00:00:00 2001 From: Technorama Ltd Date: Sun, 12 Jul 2015 21:29:00 -0400 Subject: [PATCH 1/5] Added perm parameter to File#new --- src/file.cr | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/file.cr b/src/file.cr index 0c7d7ce2666b..608e37f134f9 100644 --- a/src/file.cr +++ b/src/file.cr @@ -21,10 +21,10 @@ class File < FileDescriptorIO # :nodoc: DEFAULT_CREATE_MODE = LibC::S_IRUSR | LibC::S_IWUSR | LibC::S_IRGRP | LibC::S_IROTH - def initialize(filename, mode = "r") + def initialize(filename, mode = "r", perm = DEFAULT_CREATE_MODE) oflag = open_flag(mode) - fd = LibC.open(filename, oflag, DEFAULT_CREATE_MODE) + fd = LibC.open(filename, oflag, perm) if fd < 0 raise Errno.new("Error opening file '#{filename}' with mode '#{mode}'") end @@ -213,12 +213,12 @@ class File < FileDescriptorIO (stat.st_mode & LibC::S_IFMT) == LibC::S_IFLNK end - def self.open(filename, mode = "r") - new filename, mode + def self.open(filename, mode = "r", perm = DEFAULT_CREATE_MODE) + new filename, mode, perm end - def self.open(filename, mode = "r") - file = File.new filename, mode + def self.open(filename, mode = "r", perm = DEFAULT_CREATE_MODE) + file = File.new filename, mode, perm begin yield file ensure From fa3f051f72058a80849b5e12b5016689a7956f59 Mon Sep 17 00:00:00 2001 From: Technorama Ltd Date: Sun, 12 Jul 2015 21:32:02 -0400 Subject: [PATCH 2/5] Error checking for failure in Process#fork --- src/process/process.cr | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/process/process.cr b/src/process/process.cr index 0751a9994f52..9e36fcbb7760 100644 --- a/src/process/process.cr +++ b/src/process/process.cr @@ -6,6 +6,7 @@ lib LibC fun getpid : Int32 fun getppid : Int32 fun exit(status : Int32) : NoReturn + fun _exit(status : Int32) : NoReturn ifdef x86_64 ClockT = UInt64 @@ -69,6 +70,7 @@ module Process def self.fork pid = LibC.fork + raise Errno.new("fork") if pid == -1 pid = nil if pid == 0 pid end From 9c00324d54e71b1a90e73df821051697af9d2d27 Mon Sep 17 00:00:00 2001 From: Technorama Ltd Date: Wed, 15 Jul 2015 04:10:23 -0400 Subject: [PATCH 3/5] Add FileDescriptorIO#close_on_exec= and FileDescriptorIO#fcntl Add error checking to some fcntl calls. --- src/io.cr | 1 + src/io/file_descriptor_io.cr | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/io.cr b/src/io.cr index cd8ded8fd970..f1067ce7a363 100644 --- a/src/io.cr +++ b/src/io.cr @@ -1,5 +1,6 @@ lib LibC enum FCNTL + FD_CLOEXEC = 1 F_GETFL = 3 F_SETFL = 4 end diff --git a/src/io/file_descriptor_io.cr b/src/io/file_descriptor_io.cr index 4aa9719497d1..e2d438139f3a 100644 --- a/src/io/file_descriptor_io.cr +++ b/src/io/file_descriptor_io.cr @@ -21,8 +21,8 @@ class FileDescriptorIO @out_count = 0 unless blocking - before = LibC.fcntl(@fd, LibC::FCNTL::F_GETFL) - LibC.fcntl(@fd, LibC::FCNTL::F_SETFL, before | LibC::O_NONBLOCK) + before = fcntl(LibC::FCNTL::F_GETFL) + fcntl(LibC::FCNTL::F_SETFL, before | LibC::O_NONBLOCK) if @edge_triggerable @event = Scheduler.create_fd_events(self) end @@ -89,6 +89,16 @@ class FileDescriptorIO self end + def close_on_exec=(arg : Bool) + fcntl(LibC::FCNTL::FD_CLOEXEC, arg ? 1 : 0) + end + + def fcntl cmd, arg = 0 + r = LibC.fcntl @fd, cmd, arg + raise Errno.new("fcntl() failed") if r == -1 + r + end + private def unbuffered_read(slice : Slice(UInt8), count) loop do bytes_read = LibC.read(@fd, slice.pointer(count), LibC::SizeT.cast(count)) From 692269ccdf2e4164c258aaaea4f87339d4165294 Mon Sep 17 00:00:00 2001 From: Technorama Ltd Date: Wed, 15 Jul 2015 04:13:34 -0400 Subject: [PATCH 4/5] Add Process#spawn Refactor Process#run to use Process#spawn. --- src/process/run.cr | 74 +++++++++++++++++++------------------------- src/process/spawn.cr | 59 +++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 42 deletions(-) create mode 100644 src/process/spawn.cr diff --git a/src/process/run.cr b/src/process/run.cr index 60b246752ffd..7cc1f99ce3f6 100644 --- a/src/process/run.cr +++ b/src/process/run.cr @@ -1,68 +1,58 @@ -lib LibC - fun execvp(file : UInt8*, argv : UInt8**) : Int32 -end - -def Process.run(command, args = nil, output = nil : IO | Bool, input = nil : String | IO) - argv = [command.cstr] - if args - args.each do |arg| - argv << arg.cstr - end - end - argv << Pointer(UInt8).null - - if output - process_output, fork_output = IO.pipe(write_blocking: true) - end - - if input +# Executes a command, waits for it to exit and returns a Process::Status object. +# +# Output is captured in status.output if the output parameter is true +# +# See Process.spawn for arguments +def Process.run(command, args = nil, output = nil : IO | Bool, input = nil : String | IO | Bool, chdir = nil : String) + case input + when FileDescriptorIO, Bool, nil + # passed to spawn + when IO, String fork_input, process_input = IO.pipe(read_blocking: true) + fork_input.close_on_exec = true + process_input.close_on_exec = true + else + raise "unknown type #{input.inspect}" end - pid = fork do - if output == false - null = File.new("/dev/null", "r+") - STDOUT.reopen(null) - elsif fork_output - STDOUT.reopen(fork_output) - end - - if process_input && fork_input - process_input.close - STDIN.reopen(fork_input) - end - - LibC.execvp(command, argv.buffer) - LibC.exit 127 + case output + when FileDescriptorIO, false, nil + # passed to spawn + when IO, String, true + process_output, fork_output = IO.pipe(write_blocking: true) + process_output.close_on_exec = true + fork_output.close_on_exec = true + else + raise "unknown type #{input.inspect}" end - if pid == -1 - raise Errno.new("Error executing system command '#{command}'") + pid = spawn(command, args, input: (fork_input || input), output: (fork_output || output), chdir: chdir) do + process_input.close if process_input + process_output.close if process_output end status = Process::Status.new(pid) - if input - process_input = process_input.not_nil! + if process_input fork_input.not_nil!.close case input - when String - process_input.print input + when String, StringIO + process_input.print input.to_s process_input.close process_input = nil - when IO + when IO # not FileDescriptorIO input_io = input end end - if output + if process_output fork_output.not_nil!.close case output when true status_output = StringIO.new - when IO + when IO # not FileDescriptorIO status_output = output end end diff --git a/src/process/spawn.cr b/src/process/spawn.cr new file mode 100644 index 000000000000..388ce8ee19c0 --- /dev/null +++ b/src/process/spawn.cr @@ -0,0 +1,59 @@ +lib LibC + fun execvp(file : UInt8*, argv : UInt8**) : Int32 +end + +# Executes a command and returns it's pid. +# +# input|output|error +# nil -> share parent stdio +# false -> reopen /dev/null in child +# IO -> filedescriptor of object is reopened +# +# TODO: Missing pgroup, env, unsetenv_others, pgroup, new_pgroup (windows), :rlimit_..., close_others, file number redirection +def Process.spawn(command, args = nil, input = nil, output = nil, error = nil, chdir = nil, umask = nil) + spawn(command, args, input, output, error, chdir, umask) { nil } +end + +# :nodoc: +def Process.spawn(command, args = nil, input = nil : Nil | Bool | FileDescriptorIO, output = nil : Nil | Bool | FileDescriptorIO, error = nil : Nil | Bool | FileDescriptorIO, chdir = nil : Nil | String, umask = nil : Nil | UInt16, &block) + argv = [command.cstr] + if args + args.each do |arg| + argv << arg.cstr + end + end + argv << Pointer(UInt8).null + + pid = fork do +# TODO: wrap in begin/ensure and use _exit, not exit + File.umask(umask) if umask + + reopen_io(input, STDIN, "r") + reopen_io(output, STDOUT, "w") + reopen_io(error, STDERR, "w") + + Dir.chdir(chdir) if chdir + + yield # close file descriptors, etc. remove when close_others is implemented. + + LibC.execvp(command, argv.buffer) + LibC.exit 127 + end + + pid +end + +private def reopen_io srcio, dstio, mode + case srcio + when FileDescriptorIO + dstio.reopen(srcio) + when false + File.open("/dev/null", mode) do |file| + dstio.reopen(file) + end + when true, nil + # use same io as parent + else + raise "unknown object type #{srcio}" + end +end From 3617e440a465238f367c0f1ee9335c0e4c6f2a4e Mon Sep 17 00:00:00 2001 From: Technorama Ltd Date: Sat, 8 Aug 2015 02:54:33 -0400 Subject: [PATCH 5/5] Add Process#popen Refactor Process#run code simplified now accepts stderr as an argument Add Process::Status#close closes any pipes to the child and waits for the process to exit Add minimal documentation and additional specs --- spec/std/process_spec.cr | 38 +++++++-- src/compiler/crystal/compiler.cr | 4 +- src/process/popen.cr | 57 +++++++++++++ src/process/run.cr | 135 +++++++++++-------------------- src/process/spawn.cr | 37 +++++---- src/process/status.cr | 84 +++++++++++++++++-- src/signal.cr | 2 + 7 files changed, 237 insertions(+), 120 deletions(-) create mode 100644 src/process/popen.cr diff --git a/spec/std/process_spec.cr b/spec/std/process_spec.cr index 84644ebd7c73..191fa3ebdb24 100644 --- a/spec/std/process_spec.cr +++ b/spec/std/process_spec.cr @@ -11,7 +11,7 @@ describe Process do end it "returns status 127 if command could not be executed" do - Process.run("foobarbaz", output: true).exit.should eq(127) + Process.run("foobarbaz", output: nil).exit.should eq(127) end it "includes PID in process status " do @@ -32,24 +32,46 @@ describe Process do end it "gets output as string" do - Process.run("/bin/sh", {"-c", "echo hello"}, output: true).output.should eq("hello\n") + Process.run("/bin/sh", {"-c", "echo hello"}, output: nil).output.to_s.should eq("hello\n") end it "send input from string" do - Process.run("/bin/cat", input: "hello", output: true).output.should eq("hello") + Process.run("/bin/cat", input: StringIO.new("hello"), output: nil).output.to_s.should eq("hello") end it "send input from IO" do File.open(__FILE__, "r") do |file| - Process.run("/bin/cat", input: file, output: true).output.should eq(File.read(__FILE__)) + Process.run("/bin/cat", input: file, output: nil).output.to_s.should eq(File.read(__FILE__)) end end it "send output to IO" do io = StringIO.new - Process.run("/bin/cat", input: "hello", output: io).output.should be_nil + Process.run("/bin/cat", input: StringIO.new("hello"), output: io).output.to_s.should eq("hello") io.to_s.should eq("hello") end + + it "gets status code from successful process" do + system("true").should eq(true) + end + + it "gets status code from failed process" do + system("false").should eq(false) + end + + it "gets output as string" do + `echo hello`.should eq("hello\n") + end + end + + describe "popen" do + it "test alive?" do + status = Process.popen("sleep", ["60"]) + status.alive?.should be_true + status.kill + status.close + status.alive?.should be_false + end end describe "kill" do @@ -59,14 +81,14 @@ describe Process do end it "kills many process" do - pid1 = fork { loop {} } - pid2 = fork { loop {} } + pid1 = fork { loop { sleep 60 } } + pid2 = fork { loop { sleep 60 } } Process.kill(Signal::KILL, pid1, pid2).should eq(0) end end it "gets the pgid of a process id" do - pid = fork { loop {} } + pid = fork { loop { sleep 60 } } Process.getpgid(pid).should be_a(Int32) Process.kill(Signal::KILL, pid) end diff --git a/src/compiler/crystal/compiler.cr b/src/compiler/crystal/compiler.cr index ca575db42320..378099d9d9c2 100644 --- a/src/compiler/crystal/compiler.cr +++ b/src/compiler/crystal/compiler.cr @@ -214,7 +214,9 @@ module Crystal timing("Codegen (clang)") do quoted_object_names = object_names.map { |name| %("#{name}") }.join " " - system "#{CC} -o #{output_filename} #{quoted_object_names} #{@link_flags} #{lib_flags}" + cmd = "#{CC} -o #{output_filename} #{quoted_object_names} #{@link_flags} #{lib_flags}" + status = Process.run "/bin/sh", input: StringIO.new(cmd) + raise "exit=#{status.exit} #{cmd.inspect} failed" unless status.success? end end diff --git a/src/process/popen.cr b/src/process/popen.cr new file mode 100644 index 000000000000..13d2ec2d16f4 --- /dev/null +++ b/src/process/popen.cr @@ -0,0 +1,57 @@ +# Executes a command and returns a Process::Status object which contains pipes to stdin/stdout/stderr depending on the arguments passed. +# +# See Process.spawn for arguments +# Passing nil to input|output|error creates pipes that may be used to communicate with the child. The pipes are returned in the status object. +def Process.popen(command, args = nil, input = nil : Nil | FileDescriptorIO | Bool, output = nil : Nil | FileDescriptorIO | Bool, error = nil : Nil | FileDescriptorIO | Bool, chdir = nil : String?) + case input + when FileDescriptorIO, Bool + # passed to spawn + when nil + fork_input, process_input = IO.pipe(read_blocking: true) + fork_input.close_on_exec = true + process_input.close_on_exec = true + else + raise "unknown type #{input.inspect}" + end + + case output + when FileDescriptorIO, Bool + # passed to spawn + when nil + process_output, fork_output = IO.pipe(write_blocking: true) + process_output.close_on_exec = true + fork_output.close_on_exec = true + else + raise "unknown type #{error.inspect}" + end + + case error + when FileDescriptorIO, Bool + # passed to spawn + when nil + process_error, fork_error = IO.pipe(write_blocking: true) + process_error.close_on_exec = true + fork_error.close_on_exec = true + else + raise "unknown type #{error.inspect}" + end + + status = spawn(command, args, input: (fork_input || input), output: (fork_output || output), error: (fork_error || error), chdir: chdir) do + process_input.close if process_input + process_output.close if process_output + process_error.close if process_error + end + + fork_input.close if fork_input + fork_output.close if fork_output + fork_error.close if fork_error + + status.input = process_input || input + status.output = process_output || output + status.error = process_error || error + status.manage_input = !!process_input + status.manage_output = !!process_output + status.manage_error = !!process_error + + status +end diff --git a/src/process/run.cr b/src/process/run.cr index 7cc1f99ce3f6..eb56e3cdbc32 100644 --- a/src/process/run.cr +++ b/src/process/run.cr @@ -1,119 +1,74 @@ # Executes a command, waits for it to exit and returns a Process::Status object. # -# Output is captured in status.output if the output parameter is true -# # See Process.spawn for arguments -def Process.run(command, args = nil, output = nil : IO | Bool, input = nil : String | IO | Bool, chdir = nil : String) +# output|error is captured in status.output if the parameter is nil +# StringIO objects may also be used for input|output|error unlike popen or spawn. +def Process.run(command, args = nil, input = true : Nil | IO | StringIO | Bool, output = nil : Nil | IO | StringIO | Bool, error = true : Nil | IO | StringIO | Bool, chdir = nil : String?) case input when FileDescriptorIO, Bool, nil - # passed to spawn - when IO, String - fork_input, process_input = IO.pipe(read_blocking: true) - fork_input.close_on_exec = true - process_input.close_on_exec = true - else - raise "unknown type #{input.inspect}" + # passed to popen + popen_input = input + copy_input = nil + when IO, StringIO + # redirected to IO provided + popen_input = nil + copy_input = input end case output - when FileDescriptorIO, false, nil - # passed to spawn - when IO, String, true - process_output, fork_output = IO.pipe(write_blocking: true) - process_output.close_on_exec = true - fork_output.close_on_exec = true - else - raise "unknown type #{input.inspect}" - end - - pid = spawn(command, args, input: (fork_input || input), output: (fork_output || output), chdir: chdir) do - process_input.close if process_input - process_output.close if process_output + when FileDescriptorIO, Bool + # passed to popen + popen_output = output + copy_output = nil + when IO, StringIO, nil + # redirected to IO provided + popen_output = nil + copy_output = output || StringIO.new end - status = Process::Status.new(pid) - - if process_input - fork_input.not_nil!.close - - case input - when String, StringIO - process_input.print input.to_s - process_input.close - process_input = nil - when IO # not FileDescriptorIO - input_io = input - end + case error + when FileDescriptorIO, Bool + # passed to popen + popen_error = error + copy_error = nil + when IO, StringIO, nil + # redirected to IO provided + popen_error = nil + copy_error = error end - if process_output - fork_output.not_nil!.close - - case output - when true - status_output = StringIO.new - when IO # not FileDescriptorIO - status_output = output - end - end - - while process_input || process_output - wios = nil - rios = nil - - if process_input - wios = {process_input} - end + status = popen(command, args, input: popen_input, output: popen_output, error: popen_error, chdir: chdir) - if process_output - rios = {process_output} - end + input_copy = -> { io_copy("input", copy_input, status.input, true) } + output_copy = -> { io_copy("output", status.output, copy_output) } + error_copy = -> { io_copy("error", status.error, copy_error) } + parallel(input_copy.call, output_copy.call, error_copy.call) - buffer :: UInt8[2048] + status.close - ios = IO.select(rios, wios) - next unless ios - - if process_input && ios.includes? process_input - bytes = input_io.not_nil!.read(buffer.to_slice) - if bytes == 0 - process_input.close - process_input = nil - else - process_input.write(buffer.to_slice, bytes) - end - end - - if process_output && ios.includes? process_output - bytes = process_output.read(buffer.to_slice) - if bytes == 0 - process_output.close - process_output = nil - else - status_output.not_nil!.write(buffer.to_slice, bytes) - end - end - end - - status.exit = Process.waitpid(pid) - - if output == true - status.output = status_output.to_s - end + status.output = copy_output if copy_output + status.error = copy_error if copy_error $? = status status end +private def io_copy msg, src, dst, close_dst = false + return true unless src.is_a?(IO) && dst.is_a?(IO) + IO.copy(src, dst) + dst.close if close_dst + true # not used. compiler doesn't like nil return +end + def system(command : String) - status = Process.run("/bin/sh", input: command, output: STDOUT) + status = Process.run("/bin/sh", { "-c", command }, output: STDOUT, error: STDERR) $? = status status.success? end def `(command) - status = Process.run("/bin/sh", input: command, output: true) + status = Process.run("/bin/sh", { "-c", command }, output: nil, error: STDERR) $? = status - status.output.not_nil! + status.output.not_nil!.to_s end diff --git a/src/process/spawn.cr b/src/process/spawn.cr index 388ce8ee19c0..746e1c4cbb12 100644 --- a/src/process/spawn.cr +++ b/src/process/spawn.cr @@ -5,17 +5,17 @@ end # Executes a command and returns it's pid. # # input|output|error -# nil -> share parent stdio +# true -> share parent stdio # false -> reopen /dev/null in child -# IO -> filedescriptor of object is reopened +# IO -> filedescriptor of object is reopened in child # # TODO: Missing pgroup, env, unsetenv_others, pgroup, new_pgroup (windows), :rlimit_..., close_others, file number redirection -def Process.spawn(command, args = nil, input = nil, output = nil, error = nil, chdir = nil, umask = nil) +def Process.spawn(command, args = nil, input = true, output = true, error = true, chdir = nil, umask = nil) spawn(command, args, input, output, error, chdir, umask) { nil } end # :nodoc: -def Process.spawn(command, args = nil, input = nil : Nil | Bool | FileDescriptorIO, output = nil : Nil | Bool | FileDescriptorIO, error = nil : Nil | Bool | FileDescriptorIO, chdir = nil : Nil | String, umask = nil : Nil | UInt16, &block) +def Process.spawn(command, args = nil, input = true : Bool | FileDescriptorIO, output = true : Bool | FileDescriptorIO, error = true : Bool | FileDescriptorIO, chdir = nil : String?, umask = nil : UInt16?, &block) argv = [command.cstr] if args args.each do |arg| @@ -25,34 +25,39 @@ def Process.spawn(command, args = nil, input = nil : Nil | Bool | FileDescriptor argv << Pointer(UInt8).null pid = fork do -# TODO: wrap in begin/ensure and use _exit, not exit - File.umask(umask) if umask + begin + File.umask(umask) if umask - reopen_io(input, STDIN, "r") - reopen_io(output, STDOUT, "w") - reopen_io(error, STDERR, "w") + reopen_io(input, STDIN, "r") + reopen_io(output, STDOUT, "w") + reopen_io(error, STDERR, "w") - Dir.chdir(chdir) if chdir + Dir.chdir(chdir) if chdir - yield # close file descriptors, etc. remove when close_others is implemented. + yield # close file descriptors, etc. remove when close_others is implemented. - LibC.execvp(command, argv.buffer) - LibC.exit 127 + LibC.execvp(command, argv.buffer) + rescue ex +# TODO: print backtrace + STDERR.puts ex.inspect + ensure + LibC._exit 127 + end end - pid + Process::Status.new(pid) end private def reopen_io srcio, dstio, mode case srcio when FileDescriptorIO dstio.reopen(srcio) + when true + # use same io as parent when false File.open("/dev/null", mode) do |file| dstio.reopen(file) end - when true, nil - # use same io as parent else raise "unknown object type #{srcio}" end diff --git a/src/process/status.cr b/src/process/status.cr index ae12a5755ea6..a3270fbf875d 100644 --- a/src/process/status.cr +++ b/src/process/status.cr @@ -1,14 +1,88 @@ class Process::Status - property pid - property exit - property input - property output + getter pid + getter exit + property input, output, error :: IO | Nil + setter manage_input, manage_output, manage_error :: Bool - def initialize(@pid) + def initialize(@pid, input = nil, output = nil, error = nil) + @input = io_param(input) + @output = io_param(output) + @error = io_param(error) + + @manage_input = false + @manage_output = false + @manage_error = false end def success? @exit == 0 end + + def alive? + if @exit + false + else + begin + kill(Signal::NONE) + true + rescue ex : Errno + raise ex unless ex.errno == Errno::ESRCH + false + end + end + end + + def kill sig = Signal::TERM + Process.kill(sig, @pid) + end + + # closes any pipes to the child and waits for the process to exit + def close + i = @input + o = @output + e = @error + if @manage_input && i + i.close rescue nil + @input = nil + end + if @manage_output && o + o.close rescue nil + @output = nil + end + if @manage_error && e + e.close rescue nil + @error = nil + end + + wait + end + + def input= io + @input = io_param(io) + end + + def output= io + @output = io_param(io) + end + + def error= io + @error = io_param(io) + end + + private def wait + @exit = Process.waitpid(pid) + rescue err : Errno + raise err unless err.errno == Errno::ESRCH # process may have been reaped elsewhere or SIGCHLD ignored + end + + # drop Bool type + private def io_param io + case io + when Bool + nil + else + io + end + end end diff --git a/src/signal.cr b/src/signal.cr index 7264942f109c..fa221c52a24d 100644 --- a/src/signal.cr +++ b/src/signal.cr @@ -5,6 +5,7 @@ end ifdef darwin enum Signal + NONE = 0 HUP = 1 INT = 2 QUIT = 3 @@ -41,6 +42,7 @@ ifdef darwin end else enum Signal + NONE = 0 HUP = 1 INT = 2 QUIT = 3