From 1d3c097eec91f76bbd045f6d5d8ef226fba2540a Mon Sep 17 00:00:00 2001 From: Amit Murthy Date: Sun, 29 Oct 2017 18:48:54 +0530 Subject: [PATCH] wip : move Distributed to stdlib --- base/client.jl | 45 +-- base/deprecated.jl | 36 +++ base/exports.jl | 38 +-- base/loading.jl | 11 +- base/precompile.jl | 280 ----------------- base/sysimg.jl | 5 +- doc/make.jl | 27 +- doc/src/manual/parallel-computing.md | 34 ++- doc/src/stdlib/parallel.md | 71 +---- stdlib/Distributed/docs/src/index.md | 70 +++++ .../Distributed/src}/Distributed.jl | 11 +- .../Distributed/src}/cluster.jl | 43 ++- .../Distributed/src}/clusterserialize.jl | 2 +- .../Distributed/src}/macros.jl | 0 .../Distributed/src}/managers.jl | 0 .../Distributed/src}/messages.jl | 0 .../Distributed/src}/pmap.jl | 0 stdlib/Distributed/src/precompile.jl | 281 ++++++++++++++++++ .../Distributed/src}/process_messages.jl | 0 .../Distributed/src}/remotecall.jl | 0 .../Distributed/src}/workerpool.jl | 0 .../Distributed/test}/distributed_exec.jl | 188 ++++-------- .../Distributed/test/runtests.jl | 2 +- {test => stdlib/Distributed/test}/topology.jl | 12 +- stdlib/SharedArrays/src/SharedArrays.jl | 4 +- stdlib/SharedArrays/test/runtests.jl | 11 +- stdlib/Test/test/runtests.jl | 2 +- test/abstractarray.jl | 71 +---- test/asyncmap.jl | 60 ++++ test/channels.jl | 1 + test/choosetests.jl | 8 +- test/compile.jl | 6 +- test/generic_map_tests.jl | 84 ++++++ test/meta.jl | 3 +- test/runtests.jl | 5 +- 35 files changed, 732 insertions(+), 679 deletions(-) create mode 100644 stdlib/Distributed/docs/src/index.md rename {base/distributed => stdlib/Distributed/src}/Distributed.jl (92%) rename {base/distributed => stdlib/Distributed/src}/cluster.jl (97%) rename {base/distributed => stdlib/Distributed/src}/clusterserialize.jl (99%) rename {base/distributed => stdlib/Distributed/src}/macros.jl (100%) rename {base/distributed => stdlib/Distributed/src}/managers.jl (100%) rename {base/distributed => stdlib/Distributed/src}/messages.jl (100%) rename {base/distributed => stdlib/Distributed/src}/pmap.jl (100%) create mode 100644 stdlib/Distributed/src/precompile.jl rename {base/distributed => stdlib/Distributed/src}/process_messages.jl (100%) rename {base/distributed => stdlib/Distributed/src}/remotecall.jl (100%) rename {base/distributed => stdlib/Distributed/src}/workerpool.jl (100%) rename {test => stdlib/Distributed/test}/distributed_exec.jl (88%) rename test/distributed.jl => stdlib/Distributed/test/runtests.jl (84%) rename {test => stdlib/Distributed/test}/topology.jl (88%) create mode 100644 test/asyncmap.jl create mode 100644 test/generic_map_tests.jl diff --git a/base/client.jl b/base/client.jl index 63425d9acbdaaf..092cf9ead95d4a 100644 --- a/base/client.jl +++ b/base/client.jl @@ -287,24 +287,10 @@ function process_options(opts::JLOptions) # remove filename from ARGS global PROGRAM_FILE = arg_is_program ? shift!(ARGS) : "" - # startup worker. - # opts.startupfile, opts.load, etc should should not be processed for workers. - if opts.worker == 1 - # does not return - if opts.cookie != C_NULL - start_worker(unsafe_string(opts.cookie)) - else - start_worker() - end - end - - # add processors - if opts.nprocs > 0 - addprocs(opts.nprocs) - end - # load processes from machine file - if opts.machinefile != C_NULL - addprocs(load_machine_file(unsafe_string(opts.machinefile))) + # Load Distributed module only if any of the Distributed options have been specified. + if (opts.worker == 1) || (opts.nprocs > 0) || (opts.machinefile != C_NULL) + eval(Main, :(using Distributed)) + invokelatest(Main.Distributed.process_opts, opts) end # load ~/.juliarc file @@ -319,8 +305,12 @@ function process_options(opts::JLOptions) println() elseif cmd == 'L' # load file immediately on all processors - @sync for p in procs() - @async remotecall_wait(include, p, Main, arg) + if nprocs() == 1 + include(Main, arg) + else + @sync for p in invokelatest(Main.procs) + @async invokelatest(Main.remotecall_wait, include, p, Main, arg) + end end end end @@ -349,21 +339,6 @@ function load_juliarc() nothing end -function load_machine_file(path::AbstractString) - machines = [] - for line in split(read(path, String),'\n'; keep=false) - s = split(line, '*'; keep = false) - map!(strip, s, s) - if length(s) > 1 - cnt = isnumber(s[1]) ? parse(Int,s[1]) : Symbol(s[1]) - push!(machines,(s[2], cnt)) - else - push!(machines,line) - end - end - return machines -end - import .Terminals import .REPL diff --git a/base/deprecated.jl b/base/deprecated.jl index f05f28b10c1176..8310fc6a045c1f 100644 --- a/base/deprecated.jl +++ b/base/deprecated.jl @@ -1355,6 +1355,42 @@ export conv, conv2, deconv, filt, filt!, xcorr @deprecate_moved PollingFileWatcher "FileWatching" true true @deprecate_moved watch_file "FileWatching" true true @deprecate_moved FileMonitor "FileWatching" true true +# FIXME : This causes problems with the `myid` in loading.jl +#@deprecate_binding Distributed nothing true ", run `using Distributed` instead" + +@eval @deprecate_moved $(Symbol("@spawn")) "Distributed" true true +@eval @deprecate_moved $(Symbol("@spawnat")) "Distributed" true true +@eval @deprecate_moved $(Symbol("@fetch")) "Distributed" true true +@eval @deprecate_moved $(Symbol("@fetchfrom")) "Distributed" true true +@eval @deprecate_moved $(Symbol("@everywhere")) "Distributed" true true +@eval @deprecate_moved $(Symbol("@parallel")) "Distributed" true true + +@deprecate_moved addprocs "Distributed" true true +@deprecate_moved CachingPool "Distributed" true true +@deprecate_moved clear! "Distributed" true true +@deprecate_moved ClusterManager "Distributed" true true +@deprecate_moved default_worker_pool "Distributed" true true +@deprecate_moved init_worker "Distributed" true true +@deprecate_moved interrupt "Distributed" true true +@deprecate_moved launch "Distributed" true true +@deprecate_moved manage "Distributed" true true +@deprecate_moved nworkers "Distributed" true true +@deprecate_moved pmap "Distributed" true true +@deprecate_moved procs "Distributed" true true +@deprecate_moved remote "Distributed" true true +@deprecate_moved remotecall "Distributed" true true +@deprecate_moved remotecall_fetch "Distributed" true true +@deprecate_moved remotecall_wait "Distributed" true true +@deprecate_moved remote_do "Distributed" true true +@deprecate_moved rmprocs "Distributed" true true +@deprecate_moved workers "Distributed" true true +@deprecate_moved WorkerPool "Distributed" true true +@deprecate_moved RemoteChannel "Distributed" true true +@deprecate_moved Future "Distributed" true true +@deprecate_moved WorkerConfig "Distributed" true true +@deprecate_moved RemoteException "Distributed" true true +@deprecate_moved ProcessExitedException "Distributed" true true + @deprecate_moved crc32c "CRC32c" true true diff --git a/base/exports.jl b/base/exports.jl index 889910e51cdfca..9024406119d183 100644 --- a/base/exports.jl +++ b/base/exports.jl @@ -18,7 +18,6 @@ export Markdown, Threads, Iterators, - Distributed, # Types AbstractChannel, @@ -1262,38 +1261,7 @@ export nzrange, nnz, -# Distributed module re-exports - @spawn, - @spawnat, - @fetch, - @fetchfrom, - @everywhere, - @parallel, - - addprocs, - CachingPool, - clear!, - ClusterManager, - default_worker_pool, - init_worker, - interrupt, - launch, - manage, +# Minimal set of Distributed exports - useful for a program to check if running +# in distributed mode or not. myid, - nprocs, - nworkers, - pmap, - procs, - remote, - remotecall, - remotecall_fetch, - remotecall_wait, - remote_do, - rmprocs, - workers, - WorkerPool, - RemoteChannel, - Future, - WorkerConfig, - RemoteException, - ProcessExitedException + nprocs diff --git a/base/loading.jl b/base/loading.jl index 4e3085323b2173..79472f27c676a6 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -295,18 +295,19 @@ then tries paths in the global array `LOAD_PATH`. `require` is case-sensitive on all platforms, including those with case-insensitive filesystems like macOS and Windows. """ + +myid() = isdefined(Main, :Distributed) ? invokelatest(Main.Distributed.myid) : 1 +nprocs() = isdefined(Main, :Distributed) ? invokelatest(Main.Distributed.nprocs) : 1 + function require(mod::Symbol) if !root_module_exists(mod) _require(mod) # After successfully loading, notify downstream consumers if toplevel_load[] && myid() == 1 && nprocs() > 1 # broadcast top-level import/using from node 1 (only) - @sync for p in procs() + @sync for p in invokelatest(Main.procs) p == 1 && continue - @async remotecall_wait(p) do - require(mod) - nothing - end + @async invokelatest(Main.remotecall_wait, ()->(require(mod); nothing), p) end end for callback in package_callbacks diff --git a/base/precompile.jl b/base/precompile.jl index b047275fbd82f0..815e1d1bbd3f8d 100644 --- a/base/precompile.jl +++ b/base/precompile.jl @@ -16,9 +16,6 @@ end precompile(Tuple{Type{Base.Multimedia.TextDisplay}, Base.TTY}) precompile(Tuple{typeof(Base._start)}) precompile(Tuple{typeof(Base.copy!), Array{String, 1}, Int64, Array{Any, 1}, Int64, Int64}) -precompile(Tuple{typeof(Base.empty!), Base.Dict{Int64, Union{Base.Distributed.Worker, Base.Distributed.LocalProcess}}}) -precompile(Tuple{typeof(Core.Inference.isbits), Base.Distributed.DefaultClusterManager}) -precompile(Tuple{typeof(Base.Distributed.init_worker), String, Base.Distributed.DefaultClusterManager}) precompile(Tuple{typeof(Base.finalizer), typeof(Base.uvfinalize), Base.TCPServer}) precompile(Tuple{Type{Base.TCPServer}, Ptr{Void}, Int64}) precompile(Tuple{typeof(Base.show), Base.IOContext{Base.GenericIOBuffer{Array{UInt8, 1}}}, Int32}) @@ -44,11 +41,6 @@ precompile(Tuple{typeof(Core.Inference.length), Tuple{Core.Inference.Const, Data precompile(Tuple{typeof(Core.Inference.getindex), Tuple{Core.Inference.Const, DataType, Core.Inference.Const, Core.Inference.Const}, Int64}) precompile(Tuple{typeof(Base.sync_add), Task}) precompile(Tuple{typeof(Core.Inference.getindex), Tuple{Int64, Tuple{Int64, Tuple{}}}, Int64}) -precompile(Tuple{typeof(Base.Distributed.local_remotecall_thunk), typeof(Base.Distributed.set_valid_processes), Tuple{Array{Int64, 1}}, Array{Any, 1}}) -precompile(Tuple{typeof(Base.Distributed.remote_do), typeof(Base.Distributed.set_valid_processes), Base.Distributed.Worker, Array{Int64, 1}}) -precompile(Tuple{typeof(Base.Distributed.remote_do), typeof(Base.Distributed.set_valid_processes), Base.Distributed.LocalProcess, Array{Int64, 1}}) -precompile(Tuple{getfield(Base.Distributed, Symbol("#kw##remote_do")), Array{Any, 1}, typeof(Base.Distributed.remote_do), typeof(Base.Distributed.set_valid_processes), Base.Distributed.Worker, Array{Int64, 1}}) -precompile(Tuple{getfield(Base.Distributed, Symbol("#kw##remote_do")), Array{Any, 1}, typeof(Base.Distributed.remote_do), typeof(Base.Distributed.set_valid_processes), Base.Distributed.LocalProcess, Array{Int64, 1}}) precompile(Tuple{typeof(Base.Sort.Float.isnan), Base.Order.ForwardOrdering, Float32}) precompile(Tuple{typeof(Base.Sort.Float.isnan), Base.Order.ForwardOrdering, Float64}) precompile(Tuple{typeof(Base.Sort.Float.isnan), Base.Order.ReverseOrdering{Base.Order.ForwardOrdering}, Float32}) @@ -286,31 +278,19 @@ precompile(Tuple{typeof(Base.:(==)), Array{Char, 1}, Array{Char, 1}}) precompile(Tuple{typeof(Base.LineEdit.update_key_repeats), Base.LineEdit.MIState, Array{Char, 1}}) precompile(Tuple{typeof(Base.LineEdit.reset_state), Base.LineEdit.MIState}) precompile(Tuple{Type{Base.Dict{Any, Any}}, Array{Any, 1}}) -precompile(Tuple{typeof(Base.Distributed.default_addprocs_params)}) -precompile(Tuple{typeof(Base.Distributed.topology), Symbol}) -precompile(Tuple{typeof(Base.shift!), Array{Base.Distributed.WorkerConfig, 1}}) precompile(Tuple{typeof(Base.sync_end)}) precompile(Tuple{typeof(Base.wait), Task}) -precompile(Tuple{typeof(Base.Distributed.workers)}) precompile(Tuple{typeof(Base.sort!), Array{Int64, 1}}) precompile(Tuple{typeof(Base.lock), Base.ReentrantLock}) -precompile(Tuple{getfield(Base.Distributed, Symbol("#kw##addprocs_locked")), Array{Any, 1}, typeof(Base.Distributed.addprocs_locked), Base.Distributed.SSHManager}) precompile(Tuple{typeof(Base.unlock), Base.ReentrantLock}) -precompile(Tuple{typeof(Base.Distributed.check_addprocs_args), Array{Any, 1}}) -precompile(Tuple{Type{Base.Distributed.SSHManager}, Array{Any, 1}}) -precompile(Tuple{getfield(Base.Distributed, Symbol("#kw##addprocs")), Array{Any, 1}, typeof(Base.Distributed.addprocs), Base.Distributed.SSHManager}) precompile(Tuple{typeof(Base.cmd_gen), Tuple{Tuple{}}}) precompile(Tuple{typeof(Base.extrema), Array{Int64, 1}}) precompile(Tuple{typeof(Base.Sort.sort_int_range!), Array{Int64, 1}, Int64, Int64}) precompile(Tuple{typeof(Core.Inference.isbits), Base.Sort.QuickSortAlg}) -precompile(Tuple{getfield(Base.Distributed, Symbol("#kw##addprocs_locked")), Array{Any, 1}, typeof(Base.Distributed.addprocs_locked), Base.Distributed.LocalManager}) precompile(Tuple{typeof(Base.ht_keyindex2!), Base.Dict{Any, Any}, Symbol}) precompile(Tuple{typeof(Base._setindex!), Base.Dict{Any, Any}, Symbol, Symbol, Int64}) precompile(Tuple{Type{Base.Dict{Any, Any}}, Base.Pair{Symbol, Symbol}, Base.Pair{Symbol, String}, Base.Pair{Symbol, String}, Base.Pair{Symbol, Base.Cmd}, Base.Pair{Symbol, Bool}}) -precompile(Tuple{getfield(Base.Distributed, Symbol("#kw##addprocs")), Array{Any, 1}, typeof(Base.Distributed.addprocs), Base.Distributed.LocalManager}) precompile(Tuple{typeof(Base.listenany), Base.IPv4, UInt16}) -precompile(Tuple{typeof(Base.Distributed.check_master_connect)}) -precompile(Tuple{typeof(Base.load_machine_file), String}) precompile(Tuple{typeof(Base.load_juliarc)}) precompile(Tuple{typeof(Base.shift!), Array{String, 1}}) precompile(Tuple{typeof(Base.print), Base.IOContext{Base.GenericIOBuffer{Array{UInt8, 1}}}, String, String, Char}) @@ -380,17 +360,7 @@ precompile(Tuple{typeof(Base.Terminals.hascolor), Base.Terminals.TTYTerminal}) precompile(Tuple{Type{Base.REPL.LineEditREPL}, Base.Terminals.TTYTerminal, Bool}) precompile(Tuple{typeof(Base.close), Base.IOStream}) precompile(Tuple{typeof(Base._atexit)}) -precompile(Tuple{typeof(Base.Distributed.terminate_all_workers)}) precompile(Tuple{getfield(Base, Symbol("#kw##notify")), Array{Any, 1}, typeof(Base.notify), Base.Condition, Void}) -precompile(Tuple{typeof(Base.Distributed.local_remotecall_thunk), typeof(Base.exit), Tuple{}, Array{Any, 1}}) -precompile(Tuple{typeof(Base.Distributed.remote_do), typeof(Base.exit), Base.Distributed.Worker}) -precompile(Tuple{typeof(Base.Distributed.remote_do), typeof(Base.exit), Base.Distributed.LocalProcess}) -precompile(Tuple{getfield(Base.Distributed, Symbol("#kw##remote_do")), Array{Any, 1}, typeof(Base.Distributed.remote_do), typeof(Base.exit), Base.Distributed.Worker}) -precompile(Tuple{getfield(Base.Distributed, Symbol("#kw##remote_do")), Array{Any, 1}, typeof(Base.Distributed.remote_do), typeof(Base.exit), Base.Distributed.LocalProcess}) -precompile(Tuple{typeof(Base.Distributed.set_worker_state), Base.Distributed.Worker, Base.Distributed.WorkerState}) -precompile(Tuple{typeof(Base.Distributed.set_worker_state), Base.Distributed.LocalProcess, Base.Distributed.WorkerState}) -precompile(Tuple{getfield(Base.Distributed, Symbol("#kw##rmprocs")), Array{Any, 1}, typeof(Base.Distributed.rmprocs), Array{Int64, 1}}) -precompile(Tuple{typeof(Base.Distributed.interrupt), Array{Int64, 1}}) precompile(Tuple{typeof(Base.uvfinalize), Base.TTY}) precompile(Tuple{typeof(Base.uv_status_string), Base.TTY}) precompile(Tuple{typeof(Base._fd), Base.TTY}) @@ -407,7 +377,6 @@ precompile(Tuple{typeof(Base.check_open), Base.TTY}) precompile(Tuple{typeof(Base.stream_wait), Task}) precompile(Tuple{typeof(Base.uv_write), Base.TTY, Ptr{UInt8}, UInt64}) precompile(Tuple{typeof(Base.flush), Base.TTY}) -precompile(Tuple{typeof(Base.Distributed.flush_gc_msgs)}) precompile(Tuple{typeof(Base.__atreplinit), Base.REPL.LineEditREPL}) precompile(Tuple{typeof(Base.:(==)), Base.Multimedia.TextDisplay, Base.REPL.REPLDisplay{Base.REPL.LineEditREPL}}) precompile(Tuple{typeof(Base.:(==)), Base.REPL.REPLDisplay{Base.REPL.LineEditREPL}, Base.REPL.REPLDisplay{Base.REPL.LineEditREPL}}) @@ -1067,7 +1036,6 @@ precompile(Tuple{typeof(Base.unsafe_copy!), Array{AbstractString, 1}, Int64, Arr precompile(Tuple{typeof(Base.start), Array{AbstractString, 1}}) precompile(Tuple{typeof(Base.done), Array{AbstractString, 1}, Int64}) precompile(Tuple{typeof(Base.replace), String, Base.Regex, String}) -precompile(Tuple{typeof(Base.Distributed.addprocs), Int64}) precompile(Tuple{typeof(Base.setindex!), Base.Dict{Any, Any}, Base.Cmd, Symbol}) precompile(Tuple{typeof(Base._setindex!), Base.Dict{Any, Any}, Base.Cmd, Symbol, Int64}) precompile(Tuple{typeof(Base.setindex!), Base.Dict{Any, Any}, Bool, Symbol}) @@ -1084,8 +1052,6 @@ precompile(Tuple{typeof(Base.isopen), Base.PipeEndpoint}) precompile(Tuple{typeof(Base.stream_wait), Base.PipeEndpoint, Base.Condition}) precompile(Tuple{typeof(Base.close), Base.PipeEndpoint}) precompile(Tuple{getfield(Base, Symbol("#kw##setenv")), Array{Any, 1}, typeof(Base.setenv), Base.Cmd}) -precompile(Tuple{Type{Base.Distributed.WorkerConfig}}) -precompile(Tuple{typeof(Base.Distributed.launch), Base.Distributed.LocalManager, Base.Dict{Any, Any}, Array{Base.Distributed.WorkerConfig, 1}, Base.Condition}) precompile(Tuple{typeof(Base.julia_cmd), String}) precompile(Tuple{typeof(Base.cmd_gen), Tuple{Tuple{String}, Tuple{String, String}, Tuple{String, String}, Tuple{String, String}, Tuple{String, String}}}) precompile(Tuple{typeof(Base.arg_gen), String, String}) @@ -1138,7 +1104,6 @@ precompile(Tuple{Type{Base.TCPSocket}}) precompile(Tuple{typeof(Base.getsockname), Base.TCPSocket}) precompile(Tuple{typeof(Base.unsafe_store!), Ptr{Int32}, Int64}) precompile(Tuple{Type{Base.Multimedia.TextDisplay}, Base.PipeEndpoint}) -precompile(Tuple{typeof(Base.Distributed.start_worker), Base.PipeEndpoint, String}) precompile(Tuple{typeof(Core.Inference.isbits), OutOfMemoryError}) precompile(Tuple{typeof(Core.Inference.isbits), Tuple{Int64, Int64, Int64, Int64, Int64, Int64, Int64, Int64, Int64, Int64, Int64, Int64, Int64}}) precompile(Tuple{typeof(Base.getaddrinfo), String}) @@ -1168,11 +1133,9 @@ precompile(Tuple{typeof(Base.stream_wait), Base.TCPSocket, Base.Condition}) precompile(Tuple{typeof(Base.wait_connected), Base.TCPSocket}) precompile(Tuple{typeof(Base.print), Base.GenericIOBuffer{Array{UInt8, 1}}, String, Base.IPv4, String}) precompile(Tuple{typeof(Base.print), Base.GenericIOBuffer{Array{UInt8, 1}}, String, Base.IPv6, String}) -precompile(Tuple{typeof(Base.Distributed.socket_reuse_port)}) precompile(Tuple{typeof(Base.promote_type), Type{Int64}, Type{Int16}}) precompile(Tuple{typeof(Base.promote_rule), Type{Int16}, Type{Int64}}) precompile(Tuple{typeof(Base.promote_result), Type{Int64}, Type{Int16}, Type{Int64}, Type{Union{}}}) -precompile(Tuple{typeof(Base.Distributed.flush_gc_msgs)}) precompile(Tuple{typeof(Base.show), Base.GenericIOBuffer{Array{UInt8, 1}}, Int16}) precompile(Tuple{typeof(Base.print), Base.GenericIOBuffer{Array{UInt8, 1}}, Int16}) precompile(Tuple{typeof(Base.uv_status_string), Base.PipeServer}) @@ -1193,29 +1156,19 @@ precompile(Tuple{typeof(Base.accept), Base.TCPServer}) precompile(Tuple{typeof(Base.tryparse_internal), Type{Int16}, Base.SubString{String}, Int64, Int64, Int64, Bool}) precompile(Tuple{typeof(Base.parse), Type{Int16}, Base.SubString{String}}) precompile(Tuple{typeof(Base.write), Base.PipeEndpoint, String}) -precompile(Tuple{typeof(Base.Distributed.disable_nagle), Base.TCPServer}) -precompile(Tuple{typeof(Base.Distributed.next_tunnel_port)}) precompile(Tuple{typeof(Core.Inference.isbits), Tuple{Tuple{String}, Tuple{String}, Tuple{String}, Tuple{String}, Tuple{String}, Tuple{String}}}) precompile(Tuple{typeof(Base.cmd_gen), Tuple{Tuple{String}, Tuple{String}, Tuple{String}, Tuple{String}, Tuple{String}, Tuple{String}}}) -precompile(Tuple{typeof(Base._delete!), Base.Dict{Int64, Union{Base.Distributed.Worker, Base.Distributed.LocalProcess}}, Int64}) precompile(Tuple{typeof(Base.ndigits0z), UInt8}) precompile(Tuple{typeof(Base.dec), UInt8, Int64, Bool}) precompile(Tuple{typeof(Core.Inference.length), Tuple{Core.Inference.Const, typeof(Type), Core.Inference.Const}}) precompile(Tuple{typeof(Core.Inference.getindex), Tuple{Core.Inference.Const, typeof(Type), Core.Inference.Const}, Int64}) -precompile(Tuple{typeof(Base.Distributed.send_msg_), Base.Distributed.Worker, Base.Distributed.MsgHeader, Base.Distributed.JoinPGRPMsg, Bool}) -precompile(Tuple{typeof(Base.Distributed.send_msg_now), Base.Distributed.Worker, Base.Distributed.MsgHeader, Base.Distributed.JoinPGRPMsg}) precompile(Tuple{typeof(Base.rehash!), Base.Dict{Int64, Void}, Int64}) precompile(Tuple{typeof(Base.ht_keyindex2!), Base.Dict{Int64, Void}, Int64}) precompile(Tuple{typeof(Base._setindex!), Base.Dict{Int64, Void}, Void, Int64, Int64}) precompile(Tuple{typeof(Base.rehash!), Base.Dict{AbstractString, Base.Semaphore}, Int64}) precompile(Tuple{typeof(Base.resize!), Array{Base.Semaphore, 1}, Int64}) -precompile(Tuple{typeof(Base.Distributed.connect_w2w), Int64, Base.Distributed.WorkerConfig}) precompile(Tuple{typeof(Base.acquire), Base.Semaphore}) precompile(Tuple{typeof(Base.release), Base.Semaphore}) -precompile(Tuple{typeof(Base.Distributed.create_worker), Base.Distributed.LocalManager, Base.Distributed.WorkerConfig}) -precompile(Tuple{typeof(Base.Distributed.setup_launched_worker), Base.Distributed.LocalManager, Base.Distributed.WorkerConfig, Array{Int64, 1}}) -precompile(Tuple{typeof(Base.connect), Base.Distributed.LocalManager, Int64, Base.Distributed.WorkerConfig}) -precompile(Tuple{typeof(Base.Distributed.read_worker_host_port), Base.Pipe}) precompile(Tuple{typeof(Base.isreadable), Base.PipeEndpoint}) precompile(Tuple{typeof(Base.search), Base.GenericIOBuffer{Array{UInt8, 1}}, UInt8}) precompile(Tuple{typeof(Base.start_reading), Base.PipeEndpoint}) @@ -1226,18 +1179,15 @@ precompile(Tuple{typeof(Base._uv_hook_close), Base.Timer}) precompile(Tuple{typeof(Base.notify), Base.Condition, Base.EOFError, Bool, Bool}) precompile(Tuple{typeof(Base.unpreserve_handle), Base.Timer}) precompile(Tuple{typeof(Base.disassociate_julia_struct), Base.Timer}) -precompile(Tuple{typeof(Base.Distributed.parse_connection_info), String}) precompile(Tuple{typeof(Base._uv_hook_close), Base.Timer}) precompile(Tuple{typeof(Base.notify), Base.Condition, Base.EOFError, Bool, Bool}) precompile(Tuple{typeof(Base.unpreserve_handle), Base.Timer}) precompile(Tuple{typeof(Base.disassociate_julia_struct), Base.Timer}) precompile(Tuple{typeof(Base.convert), Type{Base.Nullable{AbstractString}}, Base.SubString{String}}) -precompile(Tuple{typeof(Base.Distributed.connect_to_worker), Base.SubString{String}, Int16}) precompile(Tuple{typeof(Base.connect!), Base.TCPSocket, Base.SubString{String}, UInt16}) precompile(Tuple{typeof(Base.notify), Base.Condition, Base.IPv4, Bool, Bool}) precompile(Tuple{typeof(Base.schedule), Task, Base.IPv4}) precompile(Tuple{typeof(Base.connect!), Base.TCPSocket, Base.IPv4, UInt16}) -precompile(Tuple{typeof(Base.Distributed.process_messages), Base.TCPSocket, Base.TCPSocket, Bool}) precompile(Tuple{typeof(Base.uv_status_string), Base.TCPSocket}) precompile(Tuple{typeof(Base._fd), Base.TCPSocket}) precompile(Tuple{typeof(Base.print), Base.GenericIOBuffer{Array{UInt8, 1}}, Base.TCPSocket}) @@ -1246,7 +1196,6 @@ precompile(Tuple{typeof(Base.convert), Type{Base.Nullable{AbstractString}}, Stri precompile(Tuple{typeof(Base.unpreserve_handle), Base.TCPSocket}) precompile(Tuple{typeof(Base.check_open), Base.TCPSocket}) precompile(Tuple{typeof(Base.stream_wait), Base.TCPSocket, Base.Condition}) -precompile(Tuple{getfield(Core, Symbol("#kw#Type")), Array{Any, 1}, Type{Base.Distributed.Worker}, Int64, Base.TCPSocket, Base.TCPSocket, Base.Distributed.LocalManager}) precompile(Tuple{typeof(Base.read_sub), Base.GenericIOBuffer{Array{UInt8, 1}}, Array{UInt8, 1}, Int64, Int64}) precompile(Tuple{typeof(Base.isreadable), Base.TCPSocket}) precompile(Tuple{typeof(Base.start_reading), Base.TCPSocket}) @@ -1256,140 +1205,48 @@ precompile(Tuple{typeof(Base.wait_readnb), Base.TCPSocket, Int64}) precompile(Tuple{typeof(Base.write), Base.GenericIOBuffer{Array{UInt8, 1}}, Base.GenericIOBuffer{Array{UInt8, 1}}}) precompile(Tuple{typeof(Base.readbytes!), Base.TCPSocket, Array{UInt8, 1}, Int64}) precompile(Tuple{typeof(Base.read), Base.TCPSocket, Int64}) -precompile(Tuple{typeof(Base.Distributed.worker_id_from_socket), Base.TCPSocket}) -precompile(Tuple{Type{Base.Distributed.ClusterSerializer{Base.TCPSocket}}, Base.TCPSocket}) precompile(Tuple{typeof(Base.unsafe_read), Base.TCPSocket, Ptr{UInt8}, UInt64}) precompile(Tuple{typeof(Base.read!), Base.TCPSocket, Array{Int64, 1}}) precompile(Tuple{typeof(Base.read), Base.TCPSocket, Type{UInt8}}) -precompile(Tuple{typeof(Base.Distributed.send_msg_), Base.Distributed.Worker, Base.Distributed.MsgHeader, Base.Distributed.ResultMsg, Bool}) -precompile(Tuple{typeof(Base.Distributed.send_msg_now), Base.Distributed.Worker, Base.Distributed.MsgHeader, Base.Distributed.ResultMsg}) -precompile(Tuple{Type{Core.Inference.Generator{I, F} where F where I}, Type{Core.Inference.Const}, Tuple{Int64, typeof(Base.Distributed.rmprocs)}}) -precompile(Tuple{Type{Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{Core.Inference.Const}}}, Type{Core.Inference.Const}, Tuple{Int64, typeof(Base.Distributed.rmprocs)}}) -precompile(Tuple{typeof(Core.Inference.convert), Type{Tuple{Int64, typeof(Base.Distributed.rmprocs)}}, Tuple{Int64, typeof(Base.Distributed.rmprocs)}}) -precompile(Tuple{typeof(Core.Inference.collect), Type{Any}, Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{Core.Inference.Const}}}) -precompile(Tuple{typeof(Core.Inference.iteratorsize), Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{Core.Inference.Const}}}) -precompile(Tuple{typeof(Core.Inference.iteratorsize), Type{Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{Core.Inference.Const}}}}) -precompile(Tuple{typeof(Core.Inference.iteratorsize), Type{Tuple{Int64, typeof(Base.Distributed.rmprocs)}}}) -precompile(Tuple{typeof(Core.Inference._collect), Type{Any}, Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{Core.Inference.Const}}, Core.Inference.HasLength}) -precompile(Tuple{typeof(Core.Inference.length), Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{Core.Inference.Const}}}) -precompile(Tuple{typeof(Core.Inference.length), Tuple{Int64, typeof(Base.Distributed.rmprocs)}}) -precompile(Tuple{typeof(Core.Inference.copy!), Array{Any, 1}, Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{Core.Inference.Const}}}) -precompile(Tuple{typeof(Core.Inference.start), Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{Core.Inference.Const}}}) -precompile(Tuple{typeof(Core.Inference.start), Tuple{Int64, typeof(Base.Distributed.rmprocs)}}) -precompile(Tuple{typeof(Core.Inference.done), Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{Core.Inference.Const}}, Int64}) -precompile(Tuple{typeof(Core.Inference.done), Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Int64}) -precompile(Tuple{typeof(Core.Inference.next), Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{Core.Inference.Const}}, Int64}) -precompile(Tuple{typeof(Core.Inference.next), Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Int64}) -precompile(Tuple{typeof(Core.Inference.getindex), Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Int64}) -precompile(Tuple{typeof(Core.Inference.start), Tuple{typeof(Base.Distributed.rmprocs), Int64}}) -precompile(Tuple{typeof(Core.Inference.indexed_next), Tuple{typeof(Base.Distributed.rmprocs), Int64}, Int64, Int64}) -precompile(Tuple{typeof(Core.Inference.getindex), Tuple{typeof(Base.Distributed.rmprocs), Int64}, Int64}) -precompile(Tuple{typeof(Base.Distributed.register_worker_streams), Base.Distributed.Worker}) -precompile(Tuple{typeof(Base.Distributed.register_worker_streams), Base.Distributed.LocalProcess}) precompile(Tuple{typeof(Base.convert), Type{IO}, Base.TCPSocket}) -precompile(Tuple{Type{Base.Distributed.ClusterSerializer{Base.TCPSocket}}, Base.TCPSocket}) -precompile(Tuple{typeof(Base.Distributed.worker_id_from_socket), Base.TCPSocket}) -precompile(Tuple{typeof(Base.convert), Type{Base.Distributed.ClusterSerializer{I} where I<:IO}, Base.Distributed.ClusterSerializer{Base.TCPSocket}}) -precompile(Tuple{typeof(Base.convert), Type{Base.Distributed.ClusterManager}, Base.Distributed.LocalManager}) -precompile(Tuple{typeof(Base.convert), Type{Base.Distributed.WorkerConfig}, Base.Distributed.WorkerConfig}) precompile(Tuple{typeof(Base.convert), Type{Base.Nullable{Base.VersionNumber}}, Base.Nullable{Base.VersionNumber}}) precompile(Tuple{typeof(Base.join), Base.GenericIOBuffer{Array{UInt8, 1}}, Tuple{Int64}, Char}) -precompile(Tuple{typeof(Base.get), Base.Dict{Any, Any}, Base.Distributed.RRID, Bool}) -precompile(Tuple{typeof(Base.ht_keyindex), Base.Dict{Any, Any}, Base.Distributed.RRID}) -precompile(Tuple{typeof(Core.Inference.isbits), Tuple{Int64, typeof(Base.Distributed.rmprocs)}}) -precompile(Tuple{Type{Core.Inference.Generator{I, F} where F where I}, Type{QuoteNode}, Tuple{Int64, typeof(Base.Distributed.rmprocs)}}) -precompile(Tuple{Type{Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{QuoteNode}}}, Type{QuoteNode}, Tuple{Int64, typeof(Base.Distributed.rmprocs)}}) -precompile(Tuple{typeof(Core.Inference.iteratorsize), Type{Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{QuoteNode}}}}) -precompile(Tuple{typeof(Core.Inference._collect), Type{Any}, Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{QuoteNode}}, Core.Inference.HasLength}) -precompile(Tuple{typeof(Core.Inference.length), Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{QuoteNode}}}) -precompile(Tuple{typeof(Core.Inference.copy!), Array{Any, 1}, Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{QuoteNode}}}) -precompile(Tuple{typeof(Core.Inference.start), Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{QuoteNode}}}) -precompile(Tuple{typeof(Core.Inference.done), Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{QuoteNode}}, Int64}) -precompile(Tuple{typeof(Core.Inference.next), Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{QuoteNode}}, Int64}) -precompile(Tuple{typeof(Base.Distributed.local_remotecall_thunk), typeof(Base.Distributed.rmprocs), Tuple{Int64}, Array{Any, 1}}) -precompile(Tuple{typeof(Base.Distributed.remote_do), typeof(Base.Distributed.rmprocs), Base.Distributed.Worker, Int64}) -precompile(Tuple{typeof(Base.Distributed.remote_do), typeof(Base.Distributed.rmprocs), Base.Distributed.LocalProcess, Int64}) -precompile(Tuple{getfield(Base.Distributed, Symbol("#kw##remote_do")), Array{Any, 1}, typeof(Base.Distributed.remote_do), typeof(Base.Distributed.rmprocs), Base.Distributed.Worker, Int64}) -precompile(Tuple{getfield(Base.Distributed, Symbol("#kw##remote_do")), Array{Any, 1}, typeof(Base.Distributed.remote_do), typeof(Base.Distributed.rmprocs), Base.Distributed.LocalProcess, Int64}) -precompile(Tuple{Type{Base.Distributed.ResultMsg}, Base.Distributed.RemoteException}) -precompile(Tuple{Type{Base.Distributed.ResultMsg}, Symbol}) -precompile(Tuple{typeof(Base.Distributed.send_msg_now), Base.TCPSocket, Base.Distributed.MsgHeader, Base.Distributed.ResultMsg}) precompile(Tuple{typeof(Base.close), Base.TCPSocket}) -precompile(Tuple{typeof(Base._delete!), Base.Dict{Int64, Union{Base.Distributed.Worker, Base.Distributed.LocalProcess}}, Int64}) -precompile(Tuple{typeof(Base.Distributed.def_rv_channel)}) precompile(Tuple{typeof(Base.convert), Type{Base.AbstractChannel}, Base.Channel{Any}}) precompile(Tuple{typeof(Base.ndigits0z), UInt8}) -precompile(Tuple{typeof(Base.setindex!), Base.Dict{Any, Any}, Base.Distributed.RemoteValue, Base.Distributed.RRID}) precompile(Tuple{typeof(Base.dec), UInt8, Int64, Bool}) precompile(Tuple{typeof(Core.Inference.mk_getfield), TypedSlot, Int64, Type{Integer}}) -precompile(Tuple{typeof(Base.ht_keyindex2!), Base.Dict{Any, Any}, Base.Distributed.RRID}) -precompile(Tuple{typeof(Base._setindex!), Base.Dict{Any, Any}, Base.Distributed.RemoteValue, Base.Distributed.RRID, Int64}) precompile(Tuple{typeof(Core.Inference.length), Tuple{Core.Inference.Const, DataType, Core.Inference.Const}}) precompile(Tuple{typeof(Core.Inference.getindex), Tuple{Core.Inference.Const, DataType, Core.Inference.Const}, Int64}) -precompile(Tuple{typeof(Base.notify), Base.Condition, Base.Distributed.ProcessExitedException, Bool, Bool}) precompile(Tuple{typeof(Base.rehash!), Base.Dict{Int64, Void}, Int64}) -precompile(Tuple{typeof(Base.Distributed.process_messages), Base.TCPSocket, Base.TCPSocket, Bool}) precompile(Tuple{typeof(Base.ht_keyindex2!), Base.Dict{Int64, Void}, Int64}) precompile(Tuple{typeof(Base._setindex!), Base.Dict{Int64, Void}, Void, Int64, Int64}) precompile(Tuple{typeof(Base.setindex!), Base.Dict{Int64, Void}, Void, Int64}) -precompile(Tuple{typeof(Base.pop!), Base.Dict{Int64, Union{Base.Distributed.Worker, Base.Distributed.LocalProcess}}, Int64, Void}) -precompile(Tuple{typeof(Base.Distributed.send_connection_hdr), Base.Distributed.Worker, Bool}) -precompile(Tuple{typeof(Base.Distributed.deregister_worker), Base.Distributed.ProcessGroup, Int64}) -precompile(Tuple{typeof(Base.Distributed.process_hdr), Base.TCPSocket, Bool}) -precompile(Tuple{typeof(Base.Distributed.deserialize_msg), Base.Distributed.ClusterSerializer{Base.TCPSocket}}) -precompile(Tuple{typeof(Base.Distributed.null_id), Base.Distributed.RRID}) -precompile(Tuple{typeof(Base.Distributed.deliver_result), Base.TCPSocket, Symbol, Base.Distributed.RRID, Base.Distributed.RemoteException}) -precompile(Tuple{typeof(Base.Distributed.disable_nagle), Base.TCPSocket}) precompile(Tuple{typeof(Base.wait_connected), Base.TCPSocket}) -precompile(Tuple{typeof(Base.Distributed.message_handler_loop), Base.TCPSocket, Base.TCPSocket, Bool}) -precompile(Tuple{typeof(Base.Distributed.process_tcp_streams), Base.TCPSocket, Base.TCPSocket, Bool}) precompile(Tuple{typeof(Base.write), Base.TCPSocket, String}) precompile(Tuple{typeof(Base.uv_write), Base.TCPSocket, Ptr{UInt8}, UInt64}) precompile(Tuple{typeof(Base.flush), Base.TCPSocket}) precompile(Tuple{typeof(Base.unsafe_write), Base.TCPSocket, Ptr{UInt8}, UInt64}) -precompile(Tuple{Type{Base.Distributed.JoinPGRPMsg}, Int64, Array{Union{Tuple{Any, Int64}, Tuple{Tuple{}, Any, Bool}}, 1}, Symbol, Bool}) precompile(Tuple{typeof(Base.write), Base.TCPSocket, Int64, Int64, Int64, Int64}) precompile(Tuple{typeof(Base.unsafe_write), Base.TCPSocket, Base.RefValue{Int64}, Int64}) -precompile(Tuple{typeof(Base.Serializer.serialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Base.Distributed.JoinPGRPMsg}) precompile(Tuple{typeof(Base.unsafe_write), Base.TCPSocket, Base.RefValue{UInt8}, Int64}) -precompile(Tuple{typeof(Base.Serializer.serialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Int64}) -precompile(Tuple{typeof(Base.Serializer.serialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Array{Union{Tuple{Any, Int64}, Tuple{Tuple{}, Any, Bool}}, 1}}) precompile(Tuple{typeof(Core.Inference.length), Tuple{Core.Inference.Const, DataType, Core.Inference.Const}}) precompile(Tuple{typeof(Core.Inference.getindex), Tuple{Core.Inference.Const, DataType, Core.Inference.Const}, Int64}) precompile(Tuple{typeof(Base.unsafe_write), Base.TCPSocket, Base.RefValue{Int32}, Int64}) precompile(Tuple{typeof(Base.Serializer.write_as_tag), Base.TCPSocket, Int32}) precompile(Tuple{typeof(Base.copy!), Array{Any, 1}, Base.MethodList}) -precompile(Tuple{typeof(Base.Serializer.should_send_whole_type), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Type{Any}}) -precompile(Tuple{typeof(Base.Serializer.serialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}, SimpleVector}) -precompile(Tuple{typeof(Base.Serializer.serialize_type_data), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Type{Any}, Bool}) -precompile(Tuple{typeof(Base.Serializer.serialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Type{Any}}) -precompile(Tuple{typeof(Base.Serializer.serialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Tuple{Int64}}) precompile(Tuple{typeof(Base.ht_keyindex), Base.Dict{UInt64, UInt64}, UInt64}) precompile(Tuple{typeof(Base.unique), Array{Symbol, 1}}) -precompile(Tuple{typeof(Base.Serializer.serialize_cycle), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Type{Union{Tuple{Any, Int64}, Tuple{Tuple{}, Any, Bool}}}}) -precompile(Tuple{typeof(Base.Serializer.serialize_type), Base.Distributed.ClusterSerializer{Base.TCPSocket}, DataType}) precompile(Tuple{typeof(Base.rehash!), Base.Dict{UInt64, UInt64}, Int64}) precompile(Tuple{typeof(Base.resize!), Array{UInt64, 1}, Int64}) precompile(Tuple{typeof(Base.ht_keyindex2!), Base.Dict{UInt64, UInt64}, UInt64}) -precompile(Tuple{typeof(Base.Serializer.serialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Bool}) -precompile(Tuple{typeof(Base.Distributed.serialize_global_from_main), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Symbol}) -precompile(Tuple{typeof(Base.Serializer.serialize_mod_names), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Module}) -precompile(Tuple{typeof(Base.Serializer.serialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Symbol}) precompile(Tuple{typeof(Core.Inference.isbits), Symbol}) -precompile(Tuple{typeof(Base.Serializer.serialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Module}) precompile(Tuple{typeof(Base.isassigned), Array{Symbol, 1}, Int64}) precompile(Tuple{typeof(Base.rehash!), Base.Dict{UInt64, Void}, Int64}) precompile(Tuple{typeof(Base.ht_keyindex2!), Base.Dict{UInt64, Void}, UInt64}) precompile(Tuple{typeof(Base._setindex!), Base.Dict{UInt64, Void}, Void, UInt64, Int64}) -precompile(Tuple{typeof(Base.Serializer.serialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Array{Any, 1}}) precompile(Tuple{typeof(Base.ht_keyindex), Base.Dict{WeakRef, Any}, TypeName}) -precompile(Tuple{typeof(Base.Serializer.serialize_cycle), Base.Distributed.ClusterSerializer{Base.TCPSocket}, TypeName}) precompile(Tuple{typeof(Base.Serializer.object_number), TypeName}) -precompile(Tuple{typeof(Base.Serializer.serialize_typename), Base.Distributed.ClusterSerializer{Base.TCPSocket}, TypeName}) -precompile(Tuple{typeof(Base.Serializer.serialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Array{Symbol, 1}}) -precompile(Tuple{typeof(Base.Serializer.serialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}, TypeName}) -precompile(Tuple{typeof(Base.Serializer.serialize_cycle_header), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Type{Union{Tuple{Any, Int64}, Tuple{Tuple{}, Any, Bool}}}}) -precompile(Tuple{typeof(Base.Serializer.serialize_any), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Type{Union{Tuple{Any, Int64}, Tuple{Tuple{}, Any, Bool}}}}) precompile(Tuple{typeof(Base.isassigned), Array{Union{Tuple{Any, Int64}, Tuple{Tuple{}, Any, Bool}}, 1}, Int64}) precompile(Tuple{typeof(Base.uvfinalize), Base.TCPSocket}) precompile(Tuple{typeof(Base.close), Base.TCPSocket}) @@ -1415,77 +1272,12 @@ precompile(Tuple{typeof(Base.read!), Base.TCPSocket, Array{UInt64, 1}}) precompile(Tuple{typeof(Base.unsafe_read), Base.TCPSocket, Ptr{UInt8}, UInt64}) precompile(Tuple{typeof(Base.read!), Base.TCPSocket, Array{Int64, 1}}) precompile(Tuple{typeof(Base.read), Base.TCPSocket, Type{UInt8}}) -precompile(Tuple{typeof(Base.Distributed.send_msg_), Base.Distributed.Worker, Base.Distributed.MsgHeader, Base.Distributed.ResultMsg, Bool}) -precompile(Tuple{typeof(Base.Distributed.send_msg_now), Base.Distributed.Worker, Base.Distributed.MsgHeader, Base.Distributed.ResultMsg}) -precompile(Tuple{Type{Core.Inference.Generator{I, F} where F where I}, Type{Core.Inference.Const}, Tuple{Int64, typeof(Base.Distributed.rmprocs)}}) -precompile(Tuple{Type{Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{Core.Inference.Const}}}, Type{Core.Inference.Const}, Tuple{Int64, typeof(Base.Distributed.rmprocs)}}) -precompile(Tuple{typeof(Core.Inference.convert), Type{Tuple{Int64, typeof(Base.Distributed.rmprocs)}}, Tuple{Int64, typeof(Base.Distributed.rmprocs)}}) -precompile(Tuple{typeof(Core.Inference.collect), Type{Any}, Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{Core.Inference.Const}}}) -precompile(Tuple{typeof(Core.Inference.iteratorsize), Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{Core.Inference.Const}}}) -precompile(Tuple{typeof(Core.Inference.iteratorsize), Type{Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{Core.Inference.Const}}}}) -precompile(Tuple{typeof(Core.Inference.iteratorsize), Type{Tuple{Int64, typeof(Base.Distributed.rmprocs)}}}) -precompile(Tuple{typeof(Core.Inference._collect), Type{Any}, Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{Core.Inference.Const}}, Core.Inference.HasLength}) -precompile(Tuple{typeof(Core.Inference.length), Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{Core.Inference.Const}}}) -precompile(Tuple{typeof(Core.Inference.length), Tuple{Int64, typeof(Base.Distributed.rmprocs)}}) -precompile(Tuple{typeof(Core.Inference.copy!), Array{Any, 1}, Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{Core.Inference.Const}}}) -precompile(Tuple{typeof(Core.Inference.start), Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{Core.Inference.Const}}}) -precompile(Tuple{typeof(Core.Inference.start), Tuple{Int64, typeof(Base.Distributed.rmprocs)}}) -precompile(Tuple{typeof(Core.Inference.done), Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{Core.Inference.Const}}, Int64}) -precompile(Tuple{typeof(Core.Inference.done), Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Int64}) -precompile(Tuple{typeof(Core.Inference.next), Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{Core.Inference.Const}}, Int64}) -precompile(Tuple{typeof(Core.Inference.next), Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Int64}) -precompile(Tuple{typeof(Core.Inference.getindex), Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Int64}) -precompile(Tuple{typeof(Core.Inference.start), Tuple{typeof(Base.Distributed.rmprocs), Int64}}) -precompile(Tuple{typeof(Core.Inference.indexed_next), Tuple{typeof(Base.Distributed.rmprocs), Int64}, Int64, Int64}) -precompile(Tuple{typeof(Core.Inference.getindex), Tuple{typeof(Base.Distributed.rmprocs), Int64}, Int64}) -precompile(Tuple{typeof(Base.Serializer.deserialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}}) -precompile(Tuple{typeof(Base.Serializer.deserialize_cycle), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Expr}) -precompile(Tuple{typeof(Base.Serializer.handle_deserialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Int32}) precompile(Tuple{typeof(Base._array_for), Type{Union{}}, Base.UnitRange{Int64}, Base.HasShape}) -precompile(Tuple{typeof(Base.Serializer.deserialize_array), Base.Distributed.ClusterSerializer{Base.TCPSocket}}) -precompile(Tuple{typeof(Base.Serializer.deserialize_datatype), Base.Distributed.ClusterSerializer{Base.TCPSocket}}) -precompile(Tuple{typeof(Base.Serializer.deserialize_expr), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Int64}) precompile(Tuple{typeof(Base.join), Base.GenericIOBuffer{Array{UInt8, 1}}, Tuple{Int64}, Char}) -precompile(Tuple{typeof(Core.Inference.isbits), Tuple{Int64, typeof(Base.Distributed.rmprocs)}}) -precompile(Tuple{Type{Core.Inference.Generator{I, F} where F where I}, Type{QuoteNode}, Tuple{Int64, typeof(Base.Distributed.rmprocs)}}) -precompile(Tuple{Type{Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{QuoteNode}}}, Type{QuoteNode}, Tuple{Int64, typeof(Base.Distributed.rmprocs)}}) -precompile(Tuple{typeof(Core.Inference.iteratorsize), Type{Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{QuoteNode}}}}) -precompile(Tuple{typeof(Core.Inference._collect), Type{Any}, Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{QuoteNode}}, Core.Inference.HasLength}) -precompile(Tuple{typeof(Core.Inference.length), Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{QuoteNode}}}) -precompile(Tuple{typeof(Core.Inference.copy!), Array{Any, 1}, Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{QuoteNode}}}) -precompile(Tuple{typeof(Core.Inference.start), Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{QuoteNode}}}) -precompile(Tuple{typeof(Core.Inference.done), Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{QuoteNode}}, Int64}) -precompile(Tuple{typeof(Core.Inference.next), Core.Inference.Generator{Tuple{Int64, typeof(Base.Distributed.rmprocs)}, Type{QuoteNode}}, Int64}) -precompile(Tuple{typeof(Base.Distributed.local_remotecall_thunk), typeof(Base.Distributed.rmprocs), Tuple{Int64}, Array{Any, 1}}) -precompile(Tuple{typeof(Base.Distributed.remote_do), typeof(Base.Distributed.rmprocs), Base.Distributed.Worker, Int64}) -precompile(Tuple{typeof(Base.Distributed.remote_do), typeof(Base.Distributed.rmprocs), Base.Distributed.LocalProcess, Int64}) -precompile(Tuple{getfield(Base.Distributed, Symbol("#kw##remote_do")), Array{Any, 1}, typeof(Base.Distributed.remote_do), typeof(Base.Distributed.rmprocs), Base.Distributed.Worker, Int64}) -precompile(Tuple{getfield(Base.Distributed, Symbol("#kw##remote_do")), Array{Any, 1}, typeof(Base.Distributed.remote_do), typeof(Base.Distributed.rmprocs), Base.Distributed.LocalProcess, Int64}) -precompile(Tuple{Type{Base.Distributed.ResultMsg}, Base.Distributed.RemoteException}) -precompile(Tuple{Type{Base.Distributed.ResultMsg}, Symbol}) -precompile(Tuple{typeof(Base.Distributed.send_msg_now), Base.TCPSocket, Base.Distributed.MsgHeader, Base.Distributed.ResultMsg}) precompile(Tuple{typeof(Base.setindex!), Base.Dict{Int64, Void}, Void, Int64}) -precompile(Tuple{typeof(Base.notify), Base.Condition, Base.Distributed.ProcessExitedException, Bool, Bool}) -precompile(Tuple{typeof(Base.pop!), Base.Dict{Int64, Union{Base.Distributed.Worker, Base.Distributed.LocalProcess}}, Int64, Void}) -precompile(Tuple{typeof(Base.Distributed.deregister_worker), Base.Distributed.ProcessGroup, Int64}) -precompile(Tuple{typeof(Base.Distributed.process_hdr), Base.TCPSocket, Bool}) -precompile(Tuple{typeof(Base.Distributed.null_id), Base.Distributed.RRID}) -precompile(Tuple{typeof(Base.Distributed.deliver_result), Base.TCPSocket, Symbol, Base.Distributed.RRID, Base.Distributed.RemoteException}) -precompile(Tuple{typeof(Base.Distributed.disable_nagle), Base.TCPSocket}) -precompile(Tuple{typeof(Base.Distributed.message_handler_loop), Base.TCPSocket, Base.TCPSocket, Bool}) -precompile(Tuple{typeof(Base.Distributed.process_tcp_streams), Base.TCPSocket, Base.TCPSocket, Bool}) -precompile(Tuple{typeof(Base.Serializer.deserialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Type{Union}}) -precompile(Tuple{typeof(Base.Serializer.deserialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Type{Module}}) -precompile(Tuple{typeof(Base.Serializer.deserialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Type{SimpleVector}}) precompile(Tuple{Type{Array{Union{Tuple{Any, Int64}, Tuple{Tuple{}, Any, Bool}}, 1}}, Tuple{Int64}}) precompile(Tuple{typeof(Base.eachindex), Array{Union{Tuple{Any, Int64}, Tuple{Tuple{}, Any, Bool}}, 1}}) -precompile(Tuple{Type{Base.Distributed.JoinPGRPMsg}, Int64, Array{Union{Tuple{Any, Int64}, Tuple{Tuple{}, Any, Bool}}, 1}, Symbol, Bool}) -precompile(Tuple{typeof(Base.Distributed.handle_msg), Base.Distributed.JoinPGRPMsg, Base.Distributed.MsgHeader, Base.TCPSocket, Base.TCPSocket, Base.VersionNumber}) -precompile(Tuple{Type{Base.Distributed.WorkerConfig}}) precompile(Tuple{typeof(Base.LinAlg.BLAS.set_num_threads), Int64}) -precompile(Tuple{typeof(Base.Distributed.send_msg_), Base.Distributed.Worker, Base.Distributed.MsgHeader, Base.Distributed.JoinCompleteMsg, Bool}) -precompile(Tuple{typeof(Base.Distributed.send_msg_now), Base.Distributed.Worker, Base.Distributed.MsgHeader, Base.Distributed.JoinCompleteMsg}) -precompile(Tuple{getfield(Core, Symbol("#kw#Type")), Array{Any, 1}, Type{Base.Distributed.Worker}, Int64, Base.TCPSocket, Base.TCPSocket, Base.Distributed.DefaultClusterManager}) precompile(Tuple{typeof(Base.eltype), Type{Base.Nullable{IO}}}) precompile(Tuple{Type{Base.Nullable{IO}}}) precompile(Tuple{typeof(Base.eltype), Type{Base.Nullable{AbstractString}}}) @@ -1506,23 +1298,14 @@ precompile(Tuple{typeof(Base.eltype), Type{Base.Nullable{Base.Dict{K, V} where V precompile(Tuple{Type{Base.Nullable{Base.Dict{K, V} where V where K}}}) precompile(Tuple{typeof(Base.eltype), Type{Base.Nullable{Array{T, N} where N where T}}}) precompile(Tuple{Type{Base.Nullable{Array{T, N} where N where T}}}) -precompile(Tuple{typeof(Base.Distributed.register_worker_streams), Base.Distributed.Worker}) -precompile(Tuple{typeof(Base.Distributed.register_worker_streams), Base.Distributed.LocalProcess}) precompile(Tuple{typeof(Base.convert), Type{IO}, Base.TCPSocket}) -precompile(Tuple{typeof(Base.convert), Type{Base.Distributed.ClusterSerializer{I} where I<:IO}, Base.Distributed.ClusterSerializer{Base.TCPSocket}}) -precompile(Tuple{typeof(Base.convert), Type{Base.Distributed.ClusterManager}, Base.Distributed.DefaultClusterManager}) -precompile(Tuple{typeof(Base.convert), Type{Base.Distributed.WorkerConfig}, Base.Distributed.WorkerConfig}) precompile(Tuple{typeof(Base.convert), Type{Base.Nullable{Base.VersionNumber}}, Base.VersionNumber}) -precompile(Tuple{typeof(Base.Distributed.send_connection_hdr), Base.Distributed.Worker, Bool}) precompile(Tuple{typeof(Base.unsafe_write), Base.TCPSocket, Ptr{UInt8}, UInt64}) -precompile(Tuple{typeof(Base.Distributed.manage), Base.Distributed.LocalManager, Int64, Base.Distributed.WorkerConfig, Symbol}) precompile(Tuple{typeof(Base.uv_write), Base.TCPSocket, Ptr{UInt8}, UInt64}) precompile(Tuple{typeof(Base.flush), Base.TCPSocket}) precompile(Tuple{typeof(Base.write), Base.TCPSocket, Int64, Int64, Int64, Int64}) precompile(Tuple{typeof(Base.unsafe_write), Base.TCPSocket, Base.RefValue{Int64}, Int64}) -precompile(Tuple{typeof(Base.Serializer.serialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Base.Distributed.JoinCompleteMsg}) precompile(Tuple{typeof(Base.unsafe_write), Base.TCPSocket, Base.RefValue{UInt8}, Int64}) -precompile(Tuple{typeof(Base.Serializer.serialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Int64}) precompile(Tuple{typeof(Base.write), Base.TCPSocket, Array{UInt8, 1}}) precompile(Tuple{typeof(Base.unsafe_read), Base.TCPSocket, Base.RefValue{Int32}, Int64}) precompile(Tuple{typeof(Base.unsafe_read), Base.TCPSocket, Base.RefValue{Int64}, Int64}) @@ -1531,33 +1314,13 @@ precompile(Tuple{typeof(Base.promote_type), Type{Int64}, Type{UInt8}}) precompile(Tuple{typeof(Base.promote_rule), Type{Int64}, Type{UInt8}}) precompile(Tuple{typeof(Base.promote_result), Type{Int64}, Type{UInt8}, Type{Union{}}, Type{Int64}}) precompile(Tuple{typeof(Base.read!), Base.TCPSocket, Array{UInt64, 1}}) -precompile(Tuple{typeof(Base.Serializer.deserialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}}) -precompile(Tuple{typeof(Base.Serializer.deserialize_cycle), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Expr}) -precompile(Tuple{typeof(Base.Serializer.handle_deserialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Int32}) precompile(Tuple{typeof(Base._array_for), Type{Union{}}, Base.UnitRange{Int64}, Base.HasShape}) -precompile(Tuple{typeof(Base.Serializer.deserialize_array), Base.Distributed.ClusterSerializer{Base.TCPSocket}}) -precompile(Tuple{typeof(Base.Serializer.deserialize_datatype), Base.Distributed.ClusterSerializer{Base.TCPSocket}}) -precompile(Tuple{typeof(Base.Serializer.deserialize_expr), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Int64}) -precompile(Tuple{typeof(Base.Serializer.deserialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Type{Int64}}) precompile(Tuple{typeof(Base.read), Base.TCPSocket, Type{Int64}}) -precompile(Tuple{Type{Base.Distributed.JoinCompleteMsg}, Int64, Int64}) -precompile(Tuple{typeof(Base.Distributed.handle_msg), Base.Distributed.JoinCompleteMsg, Base.Distributed.MsgHeader, Base.TCPSocket, Base.TCPSocket, Base.VersionNumber}) -precompile(Tuple{typeof(Base.hash), Base.Distributed.RemoteChannel{Base.Channel{Any}}, UInt64}) -precompile(Tuple{typeof(Base.ht_keyindex), Base.Dict{WeakRef, Void}, Base.Distributed.RemoteChannel{Base.Channel{Any}}}) -precompile(Tuple{typeof(Base.Distributed.remotecall_fetch), typeof(Base.Distributed.put_ref), Base.Distributed.Worker, Base.Distributed.RRID, Base.Distributed.WorkerPool}) -precompile(Tuple{typeof(Base.Distributed.remotecall_fetch), typeof(Base.Distributed.put_ref), Base.Distributed.LocalProcess, Base.Distributed.RRID, Base.Distributed.WorkerPool}) -precompile(Tuple{getfield(Base.Distributed, Symbol("#kw##remotecall_fetch")), Array{Any, 1}, typeof(Base.Distributed.remotecall_fetch), typeof(Base.Distributed.put_ref), Base.Distributed.Worker, Base.Distributed.RRID, Base.Distributed.WorkerPool}) -precompile(Tuple{getfield(Base.Distributed, Symbol("#kw##remotecall_fetch")), Array{Any, 1}, typeof(Base.Distributed.remotecall_fetch), typeof(Base.Distributed.put_ref), Base.Distributed.LocalProcess, Base.Distributed.RRID, Base.Distributed.WorkerPool}) precompile(Tuple{typeof(Base.rehash!), Base.Dict{WeakRef, Void}, Int64}) precompile(Tuple{typeof(Base.ht_keyindex2!), Base.Dict{WeakRef, Void}, WeakRef}) precompile(Tuple{typeof(Base._setindex!), Base.Dict{WeakRef, Void}, Void, WeakRef, Int64}) precompile(Tuple{typeof(Base.setindex!), Base.Dict{WeakRef, Void}, Void, WeakRef}) -precompile(Tuple{typeof(Base.finalizer), typeof(Base.Distributed.finalize_ref), Base.Distributed.RemoteChannel{Base.Channel{Any}}}) -precompile(Tuple{typeof(Base.Distributed.test_existing_ref), Base.Distributed.RemoteChannel{Base.Channel{Any}}}) -precompile(Tuple{Type{Base.Distributed.RemoteChannel{T} where T<:Base.AbstractChannel}, Int64}) precompile(Tuple{Type{Base.Channel{Int64}}, Int64}) -precompile(Tuple{Type{Base.Distributed.WorkerPool}}) -precompile(Tuple{typeof(Base.Distributed.default_worker_pool)}) precompile(Tuple{typeof(Base.get), Base.Nullable{Base.Dict{K, V} where V where K}, Base.Dict{Any, Any}}) precompile(Tuple{typeof(Base.setindex!), Base.Dict{Any, Any}, Int64, Symbol}) precompile(Tuple{typeof(Base._setindex!), Base.Dict{Any, Any}, Int64, Symbol, Int64}) @@ -1567,33 +1330,19 @@ precompile(Tuple{typeof(Base.convert), Type{Base.Nullable{Base.VersionNumber}}, precompile(Tuple{typeof(Base.put!), Base.Channel{Any}, Int64}) precompile(Tuple{typeof(Base.put_buffered), Base.Channel{Any}, Int64}) precompile(Tuple{typeof(Base.put_unbuffered), Base.Channel{Any}, Int64}) -precompile(Tuple{typeof(Base.Distributed.call_on_owner), typeof(Base.Distributed.put_ref), Base.Distributed.RemoteChannel{Base.Channel{Any}}, Base.Distributed.WorkerPool}) -precompile(Tuple{typeof(Base.Distributed.put_ref), Base.Distributed.RRID, Base.Distributed.WorkerPool}) -precompile(Tuple{typeof(Base.put!), Base.Distributed.RemoteValue, Base.Distributed.WorkerPool}) -precompile(Tuple{typeof(Base.put!), Base.Channel{Any}, Base.Distributed.WorkerPool}) -precompile(Tuple{typeof(Base.put_buffered), Base.Channel{Any}, Base.Distributed.WorkerPool}) -precompile(Tuple{typeof(Base.put_unbuffered), Base.Channel{Any}, Base.Distributed.WorkerPool}) -precompile(Tuple{typeof(Base.push!), Base.Distributed.WorkerPool, Int64}) precompile(Tuple{typeof(Base.check_channel_state), Base.Channel{Int64}}) precompile(Tuple{typeof(Base.put_buffered), Base.Channel{Int64}, Int64}) precompile(Tuple{typeof(Base.put_unbuffered), Base.Channel{Int64}, Int64}) precompile(Tuple{typeof(Base.put!), Base.Channel{Int64}, Int64}) precompile(Tuple{typeof(Base._delete!), Base.Dict{Any, Any}, Int64}) -precompile(Tuple{typeof(Base.Serializer.serialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Base.Distributed.RemoteDoMsg}) -precompile(Tuple{typeof(Base.Serializer.serialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}, typeof(Base.Distributed.set_valid_processes)}) -precompile(Tuple{typeof(Base.Serializer.serialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Tuple{Array{Int64, 1}}}) precompile(Tuple{typeof(Base.write), Base.TCPSocket, Array{Int64, 1}}) precompile(Tuple{typeof(Base.Serializer.serialize_array_data), Base.TCPSocket, Array{Int64, 1}}) -precompile(Tuple{typeof(Base.Serializer.serialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Array{Int64, 1}}) precompile(Tuple{typeof(Base.sort!), Array{Int64, 1}, Int64, Int64, Base.Sort.InsertionSortAlg, Base.Order.ForwardOrdering}) precompile(Tuple{typeof(Base.Sort.partition!), Array{Int64, 1}, Int64, Int64, Base.Order.ForwardOrdering}) precompile(Tuple{typeof(Base.sort!), Array{Int64, 1}, Int64, Int64, Base.Sort.QuickSortAlg, Base.Order.ForwardOrdering}) -precompile(Tuple{typeof(Base.Serializer.deserialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Type{typeof(Base.Distributed.set_valid_processes)}}) -precompile(Tuple{typeof(Base.Distributed.handle_msg), Base.Distributed.RemoteDoMsg, Base.Distributed.MsgHeader, Base.TCPSocket, Base.TCPSocket, Base.VersionNumber}) precompile(Tuple{typeof(Base.put!), Base.Channel{Any}, Tuple{Array{Int64, 1}, Void}}) precompile(Tuple{typeof(Base.put_buffered), Base.Channel{Any}, Tuple{Array{Int64, 1}, Void}}) precompile(Tuple{typeof(Base.put_unbuffered), Base.Channel{Any}, Tuple{Array{Int64, 1}, Void}}) -precompile(Tuple{typeof(Base.Distributed.set_valid_processes), Array{Int64, 1}}) precompile(Tuple{typeof(Base.promote_type), Type{Int64}, Type{Int64}}) precompile(Tuple{typeof(Base.sizehint!), Base.Dict{Int64, Void}, Int64}) precompile(Tuple{typeof(Base.union!), Base.Set{Int64}, Array{Int64, 1}}) @@ -1638,13 +1387,8 @@ precompile(Tuple{typeof(Base.last), Array{Int64, 1}}) precompile(Tuple{typeof(Base.LineEdit.edit_delete), Base.GenericIOBuffer{Array{UInt8, 1}}}) precompile(Tuple{typeof(Base.print), String}) precompile(Tuple{typeof(Base.vcat), Array{Int64, 1}}) -precompile(Tuple{typeof(Base.Distributed._rmprocs), Array{Int64, 1}, Float64}) precompile(Tuple{typeof(Core.Inference.eltype), Type{Array{Int64, 1}}}) precompile(Tuple{typeof(Base.vcat), Int64}) -precompile(Tuple{typeof(Base.kill), Base.Distributed.LocalManager, Int64, Base.Distributed.WorkerConfig}) -precompile(Tuple{typeof(Base.Serializer.serialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}, typeof(Base.exit)}) -precompile(Tuple{typeof(Base.Serializer.serialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Tuple{}}) -precompile(Tuple{typeof(Base.Serializer.deserialize), Base.Distributed.ClusterSerializer{Base.TCPSocket}, Type{typeof(Base.exit)}}) precompile(Tuple{typeof(Base.uvfinalize), Base.PipeEndpoint}) precompile(Tuple{typeof(Base.unpreserve_handle), Base.PipeEndpoint}) precompile(Tuple{typeof(Base.stream_wait), Base.PipeEndpoint, Base.Condition}) @@ -1654,19 +1398,12 @@ precompile(Tuple{typeof(Base.uvfinalize), Base.TCPSocket}) precompile(Tuple{typeof(Base._uv_hook_close), Base.TCPSocket}) precompile(Tuple{typeof(Base.in), Int64, Base.BitSet}) precompile(Tuple{typeof(Base._uv_hook_close), Base.Process}) -precompile(Tuple{typeof(Base.Distributed.finalize_ref), Base.Distributed.RemoteChannel{Base.Channel{Any}}}) precompile(Tuple{typeof(Base._delete!), Base.Dict{WeakRef, Void}, Int64}) -precompile(Tuple{typeof(Base.Distributed.send_del_client), Base.Distributed.RemoteChannel{Base.Channel{Any}}}) -precompile(Tuple{typeof(Base.:(==)), Base.Distributed.RemoteChannel{Base.Channel{Any}}, Base.Distributed.RemoteChannel{Base.Channel{Any}}}) precompile(Tuple{typeof(Base.delete!), Base.BitSet, Int64}) precompile(Tuple{typeof(Base.isempty), Base.BitSet}) precompile(Tuple{typeof(Base.any), Base.BitArray{1}}) precompile(Tuple{typeof(Base.uvfinalize), Base.PipeEndpoint}) precompile(Tuple{typeof(Base.uvfinalize), Base.Process}) -precompile(Tuple{getfield(Base.Distributed, Symbol("#kw##remotecall_fetch")), Array{Any, 1}, typeof(Base.Distributed.remotecall_fetch), typeof(Base.open), Base.Distributed.LocalProcess, typeof(Base.read), String}) -precompile(Tuple{getfield(Base.Distributed, Symbol("#kw##remotecall_fetch")), Array{Any, 1}, typeof(Base.Distributed.remotecall_fetch), typeof(Base.open), Base.Distributed.Worker, typeof(Base.read), String}) -precompile(Tuple{getfield(Base.Distributed, Symbol("#kw##remote_do")), Array{Any, 1}, typeof(Base.Distributed.remote_do), typeof(Base.exit), Base.Distributed.Worker}) -precompile(Tuple{getfield(Base.Distributed, Symbol("#kw##rmprocs")), Array{Any, 1}, typeof(Base.Distributed.rmprocs), Array{Int64, 1}}) precompile(Tuple{getfield(Base.Filesystem, Symbol("#kw##rm")), Array{Any, 1}, typeof(Base.Filesystem.rm), String}) precompile(Tuple{getfield(Base, Symbol("#kw##info")), Array{Any, 1}, typeof(Base.info), Base.IOStream, String}) precompile(Tuple{getfield(Base, Symbol("#kw##print_with_color")), Array{Any, 1}, typeof(Base.print_with_color), Symbol, Base.IOStream, String}) @@ -1675,7 +1412,6 @@ precompile(Tuple{getfield(Base, Symbol("#kw##spawn")), Array{Any, 1}, typeof(Bas precompile(Tuple{getfield(Base, Symbol("#kw##systemerror")), Array{Any, 1}, typeof(Base.systemerror), Symbol, Bool}) precompile(Tuple{getfield(Base, Symbol("#kw##with_output_color")), Array{Any, 1}, typeof(Base.with_output_color), typeof(Base.println), Symbol, Base.IOStream, Base.SubString{String}}) precompile(Tuple{getfield(Base, Symbol("#kw##with_output_color")), Array{Any, 1}, typeof(Base.with_output_color), typeof(Base.print), Symbol, Base.IOStream, String}) -precompile(Tuple{Type{Base.Distributed.Future}, Int64}) precompile(Tuple{Type{Base.Process}, Base.Cmd, Ptr{Void}, Base.Pipe, Base.TTY, Base.IOStream}) precompile(Tuple{Type{Base.Set{Tuple{String, Float64}}}, Tuple{Tuple{String, Float64}}}) precompile(Tuple{Type{Base.VersionNumber}, Int64, Int64, Int64, Tuple{String, Int64}, Tuple{}}) @@ -1693,17 +1429,6 @@ precompile(Tuple{typeof(Base.compilecache), String}) precompile(Tuple{typeof(Base.copy!), Array{Tuple{String, Float64}, 1}, Int64, Array{Tuple{String, Float64}, 1}, Int64, Int64}) precompile(Tuple{typeof(Base.create_expr_cache), String, String, Array{Any, 1}}) precompile(Tuple{typeof(Base._delete!), Base.Dict{Symbol, Base.Condition}, Int64}) -precompile(Tuple{typeof(Base.Distributed.flush_gc_msgs), Base.Distributed.Worker}) -precompile(Tuple{typeof(Base.Distributed.remote_do), typeof(Base.exit), Base.Distributed.Worker}) -precompile(Tuple{typeof(Base.Distributed.send_del_client), Base.Distributed.Future}) -precompile(Tuple{typeof(Base.Distributed.send_msg), Base.Distributed.Worker, Base.Distributed.MsgHeader, Base.Distributed.CallMsg{:call}}) -precompile(Tuple{typeof(Base.Distributed.send_msg_), Base.Distributed.Worker, Base.Distributed.MsgHeader, Base.Distributed.CallMsg{:call}, Bool}) -precompile(Tuple{typeof(Base.Distributed.send_msg), Base.Distributed.Worker, Base.Distributed.MsgHeader, Base.Distributed.CallMsg{:call_fetch}}) -precompile(Tuple{typeof(Base.Distributed.send_msg_), Base.Distributed.Worker, Base.Distributed.MsgHeader, Base.Distributed.CallMsg{:call_fetch}, Bool}) -precompile(Tuple{typeof(Base.Distributed.send_msg), Base.Distributed.Worker, Base.Distributed.MsgHeader, Base.Distributed.RemoteDoMsg}) -precompile(Tuple{typeof(Base.Distributed.send_msg_), Base.Distributed.Worker, Base.Distributed.MsgHeader, Base.Distributed.RemoteDoMsg, Bool}) -precompile(Tuple{typeof(Base.Distributed.terminate_all_workers)}) -precompile(Tuple{typeof(Base.Distributed.test_existing_ref), Base.Distributed.Future}) precompile(Tuple{typeof(Base.Docs.docm), LineNumberNode, Module, String, Expr}) precompile(Tuple{typeof(Base.Docs.docm), LineNumberNode, Module, String, Expr, Bool}) precompile(Tuple{typeof(Base.Docs.keyworddoc), String, Base.BaseDocs.Keyword}) @@ -1714,14 +1439,12 @@ precompile(Tuple{typeof(Base.Filesystem.mkdir), String, UInt16}) precompile(Tuple{typeof(Base.Filesystem.mkpath), String, UInt16}) precompile(Tuple{typeof(Base.Filesystem.samefile), String, String}) precompile(Tuple{typeof(Base.Filesystem.unlink), String}) -precompile(Tuple{typeof(Base.finalizer), typeof(Base.Distributed.finalize_ref), Base.Distributed.Future}) precompile(Tuple{typeof(Base.find_all_in_cache_path), Symbol}) precompile(Tuple{typeof(Base.find_package), String}) precompile(Tuple{typeof(Base.find_source_file), String}) precompile(Tuple{typeof(Base.getindex), Base.ObjectIdDict, Symbol}) precompile(Tuple{typeof(Base.getindex), Type{Tuple{String, Float64}}, Tuple{String, Float64}}) precompile(Tuple{typeof(Base.Grisu._show), Base.IOContext{Base.GenericIOBuffer{Array{UInt8, 1}}}, Float64, Int64, Int64, Bool, Bool}) -precompile(Tuple{typeof(Base.hash), Base.Distributed.Future, UInt64}) precompile(Tuple{typeof(Base.hash), Tuple{String, Float64}, UInt64}) precompile(Tuple{typeof(Base.ht_keyindex2!), Base.Dict{Symbol, Base.Condition}, Symbol}) precompile(Tuple{typeof(Base.ht_keyindex2!), Base.Dict{Symbol, UInt64}, Symbol}) @@ -1729,7 +1452,6 @@ precompile(Tuple{typeof(Base.ht_keyindex2!), Base.Dict{Tuple{String, Float64}, V precompile(Tuple{typeof(Base.ht_keyindex), Base.Dict{Symbol, Base.Condition}, Symbol}) precompile(Tuple{typeof(Base.ht_keyindex), Base.Dict{Symbol, UInt64}, Symbol}) precompile(Tuple{typeof(Base.ht_keyindex), Base.Dict{Tuple{String, Float64}, Void}, Tuple{String, Float64}}) -precompile(Tuple{typeof(Base.ht_keyindex), Base.Dict{WeakRef, Void}, Base.Distributed.Future}) precompile(Tuple{typeof(Base.ident_cmp), Tuple{String, String, Int64}, Tuple{String, Int64}}) precompile(Tuple{typeof(Base.include_relative), String}) precompile(Tuple{typeof(Base._include_from_serialized), String}) @@ -1831,7 +1553,6 @@ precompile(Tuple{typeof(Base.stale_cachefile), String, String}) precompile(Tuple{typeof(Base.start), Tuple{Symbol, UInt64}}) precompile(Tuple{typeof(Base.start), Tuple{Void, Void}}) precompile(Tuple{typeof(Base.:(>)), String, String}) -precompile(Tuple{typeof(Base.sync_add), Base.Distributed.Future}) precompile(Tuple{typeof(Base.trunc), Float64, Int64, Int64}) precompile(Tuple{typeof(Base.union!), Base.Set{Tuple{String, Float64}}, Tuple{Tuple{String, Float64}}}) precompile(Tuple{typeof(Base.unique_from), Array{Any, 1}, Array{Tuple{String, Float64}, 1}, Base.Set{Tuple{String, Float64}}, Int64}) @@ -1863,4 +1584,3 @@ precompile(Tuple{typeof(Core.Inference.next), Tuple{Symbol, Expr}, Int64}) precompile(Tuple{typeof(Core.Inference.start), Core.Inference.Generator{Tuple{Symbol, Expr}, Type{Core.Inference.Const}}}) precompile(Tuple{typeof(Core.Inference.start), Tuple{Expr, Int64}}) precompile(Tuple{typeof(Core.Inference.start), Tuple{Symbol, Expr}}) -precompile(Tuple{Type{Union{}}, Base.Distributed.RRID}) diff --git a/base/sysimg.jl b/base/sysimg.jl index 0a3963d4872a26..d3b2062e740e2a 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -410,9 +410,6 @@ using .SparseArrays include("asyncmap.jl") -include("distributed/Distributed.jl") -using .Distributed - # worker threads include("threadcall.jl") @@ -455,7 +452,6 @@ function __init__() Multimedia.reinit_displays() # since Multimedia.displays uses STDOUT as fallback early_init() init_load_path() - Distributed.init_parallel() init_threadcall() end @@ -473,6 +469,7 @@ unshift!(Base._included_files, (@__MODULE__, joinpath(@__DIR__, "sysimg.jl"))) # load some stdlib packages but don't put their names in Main Base.require(:DelimitedFiles) Base.require(:Test) +Base.require(:Distributed) empty!(LOAD_PATH) diff --git a/doc/make.jl b/doc/make.jl index a7b5bdeed75f0d..15d68f78622f44 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -22,22 +22,24 @@ cp_q(src, dest) = isfile(dest) || cp(src, dest) # make links for stdlib package docs if Sys.iswindows() cp_q("../stdlib/DelimitedFiles/docs/src/index.md", "src/stdlib/delimitedfiles.md") - cp_q("../stdlib/Test/docs/src/index.md", "src/stdlib/test.md") - cp_q("../stdlib/Mmap/docs/src/index.md", "src/stdlib/mmap.md") - cp_q("../stdlib/SharedArrays/docs/src/index.md", "src/stdlib/sharedarrays.md") - cp_q("../stdlib/Profile/docs/src/index.md", "src/stdlib/profile.md") - cp_q("../stdlib/Base64/docs/src/index.md", "src/stdlib/base64.md") - cp_q("../stdlib/FileWatching/docs/src/index.md", "src/stdlib/filewatching.md") + cp_q("../stdlib/Test/docs/src/index.md", "src/stdlib/test.md") + cp_q("../stdlib/Mmap/docs/src/index.md", "src/stdlib/mmap.md") + cp_q("../stdlib/SharedArrays/docs/src/index.md", "src/stdlib/sharedarrays.md") + cp_q("../stdlib/Profile/docs/src/index.md", "src/stdlib/profile.md") + cp_q("../stdlib/Base64/docs/src/index.md", "src/stdlib/base64.md") + cp_q("../stdlib/FileWatching/docs/src/index.md", "src/stdlib/filewatching.md") cp_q("../stdlib/CRC32c/docs/src/index.md", "src/stdlib/crc32c.md") + cp_q("../stdlib/Distributed/docs/src/index.md", "src/stdlib/distributed.md") else symlink_q("../../../stdlib/DelimitedFiles/docs/src/index.md", "src/stdlib/delimitedfiles.md") - symlink_q("../../../stdlib/Test/docs/src/index.md", "src/stdlib/test.md") - symlink_q("../../../stdlib/Mmap/docs/src/index.md", "src/stdlib/mmap.md") - symlink_q("../../../stdlib/SharedArrays/docs/src/index.md", "src/stdlib/sharedarrays.md") - symlink_q("../../../stdlib/Profile/docs/src/index.md", "src/stdlib/profile.md") - symlink_q("../../../stdlib/Base64/docs/src/index.md", "src/stdlib/base64.md") - symlink_q("../../../stdlib/FileWatching/docs/src/index.md", "src/stdlib/filewatching.md") + symlink_q("../../../stdlib/Test/docs/src/index.md", "src/stdlib/test.md") + symlink_q("../../../stdlib/Mmap/docs/src/index.md", "src/stdlib/mmap.md") + symlink_q("../../../stdlib/SharedArrays/docs/src/index.md", "src/stdlib/sharedarrays.md") + symlink_q("../../../stdlib/Profile/docs/src/index.md", "src/stdlib/profile.md") + symlink_q("../../../stdlib/Base64/docs/src/index.md", "src/stdlib/base64.md") + symlink_q("../../../stdlib/FileWatching/docs/src/index.md", "src/stdlib/filewatching.md") symlink_q("../../../stdlib/CRC32c/docs/src/index.md", "src/stdlib/crc32c.md") + symlink_q("../../../stdlib/Distributed/docs/src/index.md", "src/stdlib/distributed.md") end const PAGES = [ @@ -90,6 +92,7 @@ const PAGES = [ "stdlib/strings.md", "stdlib/arrays.md", "stdlib/parallel.md", + "stdlib/distributed.md", "stdlib/linalg.md", "stdlib/constants.md", "stdlib/file.md", diff --git a/doc/src/manual/parallel-computing.md b/doc/src/manual/parallel-computing.md index 8e58a29050d9a7..a9bec22db776b7 100644 --- a/doc/src/manual/parallel-computing.md +++ b/doc/src/manual/parallel-computing.md @@ -1,4 +1,7 @@ -# Parallel Computing +# Distributed Memory Parallel Computing + +An implementation of distributed memory parallel computing is provided by module `Distributed` +as part of the standard library shipped with Julia. Most modern computers possess more than one CPU, and several computers can be combined together in a cluster. Harnessing the power of these multiple CPUs allows many computations to be completed @@ -17,7 +20,7 @@ manage only one process in a two-process operation. Furthermore, these operation not look like "message send" and "message receive" but rather resemble higher-level operations like calls to user functions. -Parallel programming in Julia is built on two primitives: *remote references* and *remote calls*. +Distributed programming in Julia is built on two primitives: *remote references* and *remote calls*. A remote reference is an object that can be used from any process to refer to an object stored on a particular process. A remote call is a request by one process to call a certain function on certain arguments on another (possibly the same) process. @@ -38,7 +41,9 @@ to as "workers". When there is only one process, process 1 is considered a worke workers are considered to be all processes other than process 1. Let's try this out. Starting with `julia -p n` provides `n` worker processes on the local machine. -Generally it makes sense for `n` to equal the number of CPU cores on the machine. +Generally it makes sense for `n` to equal the number of CPU cores on the machine. Note that the `-p` +argument implicitly loads module `Distributed`. + ```julia $ ./julia -p 2 @@ -196,6 +201,18 @@ The base Julia installation has in-built support for two types of clusters: Functions [`addprocs`](@ref), [`rmprocs`](@ref), [`workers`](@ref), and others are available as a programmatic means of adding, removing and querying the processes in a cluster. +```julia-repl +julia> using Distributed + +julia> addprocs(2) +2-element Array{Int64,1}: + 2 + 3 +``` + +Module `Distributed` must be explicitly loaded on the master process before invoking [`addprocs`](@ref). +It is automatically made available on the worker processes. + Note that workers do not run a `.juliarc.jl` startup script, nor do they synchronize their global state (such as global variables, new method definitions, and loaded modules) with any of the other running processes. @@ -205,9 +222,9 @@ below in the [ClusterManagers](@ref) section. ## Data Movement -Sending messages and moving data constitute most of the overhead in a parallel program. Reducing +Sending messages and moving data constitute most of the overhead in a distributed program. Reducing the number of messages and the amount of data sent is critical to achieving performance and scalability. -To this end, it is important to understand the data movement performed by Julia's various parallel +To this end, it is important to understand the data movement performed by Julia's various distributed programming constructs. [`fetch`](@ref) can be considered an explicit data movement operation, since it directly asks @@ -401,6 +418,8 @@ Parallel for loops like these must be avoided. Fortunately, [Shared Arrays](@ref to get around this limitation: ```julia +using SharedArrays + a = SharedArray{Float64}(10) @parallel for i = 1:10 a[i] = i @@ -830,6 +849,9 @@ chunk; in contrast, in a [`SharedArray`](@ref) each "participating" process has entire array. A [`SharedArray`](@ref) is a good choice when you want to have a large amount of data jointly accessible to two or more processes on the same machine. +Shared Array support is available via module `SharedArrays` which must be explicitly loaded on +all participating workers. + [`SharedArray`](@ref) indexing (assignment and accessing values) works just as with regular arrays, and is efficient because the underlying memory is available to the local process. Therefore, most algorithms work naturally on [`SharedArray`](@ref)s, albeit in single-process mode. In cases @@ -855,6 +877,8 @@ portion of the array, thereby parallelizing initialization. Here's a brief example: ```julia-repl +julia> using Distributed + julia> addprocs(3) 3-element Array{Int64,1}: 2 diff --git a/doc/src/stdlib/parallel.md b/doc/src/stdlib/parallel.md index 3706eb2c8c7568..ebcee4fcb3b322 100644 --- a/doc/src/stdlib/parallel.md +++ b/doc/src/stdlib/parallel.md @@ -1,4 +1,4 @@ -# Tasks and Parallel Computing +# Tasks and multi-threading ## Tasks @@ -29,59 +29,6 @@ Base.asyncmap Base.asyncmap! ``` -## General Parallel Computing Support - -```@docs -Base.addprocs -Base.nprocs -Base.nworkers -Base.procs() -Base.procs(::Integer) -Base.workers -Base.rmprocs -Base.interrupt -Base.myid -Base.pmap -Base.RemoteException -Base.Future -Base.RemoteChannel(::Integer) -Base.RemoteChannel(::Function, ::Integer) -Base.wait -Base.fetch(::Any) -Base.remotecall(::Any, ::Integer, ::Any...) -Base.remotecall_wait(::Any, ::Integer, ::Any...) -Base.remotecall_fetch(::Any, ::Integer, ::Any...) -Base.remote_do(::Any, ::Integer, ::Any...) -Base.put!(::RemoteChannel, ::Any...) -Base.put!(::Future, ::Any) -Base.take!(::RemoteChannel, ::Any...) -Base.isready(::RemoteChannel, ::Any...) -Base.isready(::Future) -Base.WorkerPool -Base.CachingPool -Base.default_worker_pool -Base.clear!(::CachingPool) -Base.remote -Base.remotecall(::Any, ::Base.Distributed.AbstractWorkerPool, ::Any...) -Base.remotecall_wait(::Any, ::Base.Distributed.AbstractWorkerPool, ::Any...) -Base.remotecall_fetch(::Any, ::Base.Distributed.AbstractWorkerPool, ::Any...) -Base.remote_do(::Any, ::Base.Distributed.AbstractWorkerPool, ::Any...) -Base.timedwait -Base.@spawn -Base.@spawnat -Base.@fetch -Base.@fetchfrom -Base.@async -Base.@sync -Base.@parallel -Base.@everywhere -Base.clear!(::Any, ::Any; ::Any) -Base.remoteref_id -Base.channel_from_id -Base.worker_id_from_socket -Base.cluster_cookie() -Base.cluster_cookie(::Any) -``` ## Multi-Threading @@ -129,19 +76,3 @@ Base.acquire Base.release ``` -## Cluster Manager Interface - -This interface provides a mechanism to launch and manage Julia workers on different cluster environments. -There are two types of managers present in Base: `LocalManager`, for launching additional workers on the -same host, and `SSHManager`, for launching on remote hosts via `ssh`. TCP/IP sockets are used to connect -and transport messages between processes. It is possible for Cluster Managers to provide a different transport. - -```@docs -Base.launch -Base.manage -Base.kill(::ClusterManager, ::Int, ::WorkerConfig) -Base.connect(::ClusterManager, ::Int, ::WorkerConfig) -Base.init_worker -Base.start_worker -Base.process_messages -``` diff --git a/stdlib/Distributed/docs/src/index.md b/stdlib/Distributed/docs/src/index.md new file mode 100644 index 00000000000000..b33e7ced32e096 --- /dev/null +++ b/stdlib/Distributed/docs/src/index.md @@ -0,0 +1,70 @@ +# Distributed Computing + +```@docs +Base.addprocs +Base.nprocs +Base.nworkers +Base.procs() +Base.procs(::Integer) +Base.workers +Base.rmprocs +Base.interrupt +Base.myid +Base.pmap +Base.RemoteException +Base.Future +Base.RemoteChannel(::Integer) +Base.RemoteChannel(::Function, ::Integer) +Base.wait +Base.fetch(::Any) +Base.remotecall(::Any, ::Integer, ::Any...) +Base.remotecall_wait(::Any, ::Integer, ::Any...) +Base.remotecall_fetch(::Any, ::Integer, ::Any...) +Base.remote_do(::Any, ::Integer, ::Any...) +Base.put!(::RemoteChannel, ::Any...) +Base.put!(::Future, ::Any) +Base.take!(::RemoteChannel, ::Any...) +Base.isready(::RemoteChannel, ::Any...) +Base.isready(::Future) +Base.WorkerPool +Base.CachingPool +Base.default_worker_pool +Base.clear!(::CachingPool) +Base.remote +Base.remotecall(::Any, ::Base.Distributed.AbstractWorkerPool, ::Any...) +Base.remotecall_wait(::Any, ::Base.Distributed.AbstractWorkerPool, ::Any...) +Base.remotecall_fetch(::Any, ::Base.Distributed.AbstractWorkerPool, ::Any...) +Base.remote_do(::Any, ::Base.Distributed.AbstractWorkerPool, ::Any...) +Base.timedwait +Base.@spawn +Base.@spawnat +Base.@fetch +Base.@fetchfrom +Base.@async +Base.@sync +Base.@parallel +Base.@everywhere +Base.clear!(::Any, ::Any; ::Any) +Base.remoteref_id +Base.channel_from_id +Base.worker_id_from_socket +Base.cluster_cookie() +Base.cluster_cookie(::Any) +``` + +## Cluster Manager Interface + +This interface provides a mechanism to launch and manage Julia workers on different cluster environments. +There are two types of managers present in Base: `LocalManager`, for launching additional workers on the +same host, and `SSHManager`, for launching on remote hosts via `ssh`. TCP/IP sockets are used to connect +and transport messages between processes. It is possible for Cluster Managers to provide a different transport. + +```@docs +Base.launch +Base.manage +Base.kill(::ClusterManager, ::Int, ::WorkerConfig) +Base.connect(::ClusterManager, ::Int, ::WorkerConfig) +Base.init_worker +Base.start_worker +Base.process_messages +``` diff --git a/base/distributed/Distributed.jl b/stdlib/Distributed/src/Distributed.jl similarity index 92% rename from base/distributed/Distributed.jl rename to stdlib/Distributed/src/Distributed.jl index 0545c714fafc7c..5c902d46725e97 100644 --- a/base/distributed/Distributed.jl +++ b/stdlib/Distributed/src/Distributed.jl @@ -1,5 +1,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license +__precompile__(true) + """ Tools for distributed parallel processing. """ @@ -35,8 +37,8 @@ export interrupt, launch, manage, - myid, - nprocs, +# myid, # accessed via Base +# nprocs, # accessed via Base nworkers, pmap, procs, @@ -54,8 +56,6 @@ export RemoteException, ProcessExitedException, -# Add the following into Base as some Packages access them via Base. -# Also documented as such. process_messages, remoteref_id, channel_from_id, @@ -75,5 +75,8 @@ include("macros.jl") # @spawn and friends include("workerpool.jl") include("pmap.jl") include("managers.jl") # LocalManager and SSHManager +include("precompile.jl") + +__init__() = init_parallel() end diff --git a/base/distributed/cluster.jl b/stdlib/Distributed/src/cluster.jl similarity index 97% rename from base/distributed/cluster.jl rename to stdlib/Distributed/src/cluster.jl index e05ca8e7f84e24..77ffda71d8494e 100644 --- a/base/distributed/cluster.jl +++ b/stdlib/Distributed/src/cluster.jl @@ -896,6 +896,8 @@ function _rmprocs(pids, waitfor) end +struct ProcessExitedException <: Exception end + """ ProcessExitedException() @@ -903,7 +905,6 @@ After a client Julia process has exited, further attempts to reference the dead throw this exception. """ ProcessExitedException() -struct ProcessExitedException <: Exception end worker_from_id(i) = worker_from_id(PGRP, i) function worker_from_id(pg::ProcessGroup, i) @@ -1129,3 +1130,43 @@ function init_parallel() end write_cookie(io::IO) = write(io.in, string(cluster_cookie(), "\n")) + +function process_opts(opts) + # startup worker. + # opts.startupfile, opts.load, etc should should not be processed for workers. + if opts.worker == 1 + # does not return + if opts.cookie != C_NULL + start_worker(unsafe_string(opts.cookie)) + else + start_worker() + end + end + + # add processors + if opts.nprocs > 0 + addprocs(opts.nprocs) + end + + # load processes from machine file + if opts.machinefile != C_NULL + addprocs(load_machine_file(unsafe_string(opts.machinefile))) + end + return nothing +end + + +function load_machine_file(path::AbstractString) + machines = [] + for line in split(read(path, String),'\n'; keep=false) + s = split(line, '*'; keep = false) + map!(strip, s, s) + if length(s) > 1 + cnt = isnumber(s[1]) ? parse(Int,s[1]) : Symbol(s[1]) + push!(machines,(s[2], cnt)) + else + push!(machines,line) + end + end + return machines +end diff --git a/base/distributed/clusterserialize.jl b/stdlib/Distributed/src/clusterserialize.jl similarity index 99% rename from base/distributed/clusterserialize.jl rename to stdlib/Distributed/src/clusterserialize.jl index 4a61f59710a2e6..d09a60e9d900b2 100644 --- a/base/distributed/clusterserialize.jl +++ b/stdlib/Distributed/src/clusterserialize.jl @@ -20,7 +20,7 @@ mutable struct ClusterSerializer{I<:IO} <: AbstractSerializer anonfunc_id::UInt64 function ClusterSerializer{I}(io::I) where I<:IO - new(io, 0, ObjectIdDict(), Int[], Base.worker_id_from_socket(io), + new(io, 0, ObjectIdDict(), Int[], worker_id_from_socket(io), Set{UInt64}(), Dict{UInt64, UInt64}(), Dict{UInt64, Vector{Symbol}}(), 0) end end diff --git a/base/distributed/macros.jl b/stdlib/Distributed/src/macros.jl similarity index 100% rename from base/distributed/macros.jl rename to stdlib/Distributed/src/macros.jl diff --git a/base/distributed/managers.jl b/stdlib/Distributed/src/managers.jl similarity index 100% rename from base/distributed/managers.jl rename to stdlib/Distributed/src/managers.jl diff --git a/base/distributed/messages.jl b/stdlib/Distributed/src/messages.jl similarity index 100% rename from base/distributed/messages.jl rename to stdlib/Distributed/src/messages.jl diff --git a/base/distributed/pmap.jl b/stdlib/Distributed/src/pmap.jl similarity index 100% rename from base/distributed/pmap.jl rename to stdlib/Distributed/src/pmap.jl diff --git a/stdlib/Distributed/src/precompile.jl b/stdlib/Distributed/src/precompile.jl new file mode 100644 index 00000000000000..2a65f3b51e2b36 --- /dev/null +++ b/stdlib/Distributed/src/precompile.jl @@ -0,0 +1,281 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +precompile(Tuple{typeof(Base.empty!), Base.Dict{Int64, Union{Distributed.Worker, Distributed.LocalProcess}}}) +precompile(Tuple{typeof(Core.Inference.isbits), Distributed.DefaultClusterManager}) +precompile(Tuple{typeof(Distributed.init_worker), String, Distributed.DefaultClusterManager}) +precompile(Tuple{typeof(Distributed.local_remotecall_thunk), typeof(Distributed.set_valid_processes), Tuple{Array{Int64, 1}}, Array{Any, 1}}) +precompile(Tuple{typeof(Distributed.remote_do), typeof(Distributed.set_valid_processes), Distributed.Worker, Array{Int64, 1}}) +precompile(Tuple{typeof(Distributed.remote_do), typeof(Distributed.set_valid_processes), Distributed.LocalProcess, Array{Int64, 1}}) +precompile(Tuple{getfield(Distributed, Symbol("#kw##remote_do")), Array{Any, 1}, typeof(Distributed.remote_do), typeof(Distributed.set_valid_processes), Distributed.Worker, Array{Int64, 1}}) +precompile(Tuple{getfield(Distributed, Symbol("#kw##remote_do")), Array{Any, 1}, typeof(Distributed.remote_do), typeof(Distributed.set_valid_processes), Distributed.LocalProcess, Array{Int64, 1}}) +precompile(Tuple{typeof(Distributed.default_addprocs_params)}) +precompile(Tuple{typeof(Distributed.topology), Symbol}) +precompile(Tuple{typeof(Base.shift!), Array{Distributed.WorkerConfig, 1}}) +precompile(Tuple{typeof(Distributed.workers)}) +precompile(Tuple{getfield(Distributed, Symbol("#kw##addprocs_locked")), Array{Any, 1}, typeof(Distributed.addprocs_locked), Distributed.SSHManager}) +precompile(Tuple{typeof(Distributed.check_addprocs_args), Array{Any, 1}}) +precompile(Tuple{Type{Distributed.SSHManager}, Array{Any, 1}}) +precompile(Tuple{getfield(Distributed, Symbol("#kw##addprocs")), Array{Any, 1}, typeof(Distributed.addprocs), Distributed.SSHManager}) +precompile(Tuple{getfield(Distributed, Symbol("#kw##addprocs_locked")), Array{Any, 1}, typeof(Distributed.addprocs_locked), Distributed.LocalManager}) +precompile(Tuple{getfield(Distributed, Symbol("#kw##addprocs")), Array{Any, 1}, typeof(Distributed.addprocs), Distributed.LocalManager}) +precompile(Tuple{typeof(Distributed.check_master_connect)}) +precompile(Tuple{typeof(Distributed.terminate_all_workers)}) +precompile(Tuple{typeof(Distributed.local_remotecall_thunk), typeof(Base.exit), Tuple{}, Array{Any, 1}}) +precompile(Tuple{typeof(Distributed.remote_do), typeof(Base.exit), Distributed.Worker}) +precompile(Tuple{typeof(Distributed.remote_do), typeof(Base.exit), Distributed.LocalProcess}) +precompile(Tuple{getfield(Distributed, Symbol("#kw##remote_do")), Array{Any, 1}, typeof(Distributed.remote_do), typeof(Base.exit), Distributed.Worker}) +precompile(Tuple{getfield(Distributed, Symbol("#kw##remote_do")), Array{Any, 1}, typeof(Distributed.remote_do), typeof(Base.exit), Distributed.LocalProcess}) +precompile(Tuple{typeof(Distributed.set_worker_state), Distributed.Worker, Distributed.WorkerState}) +precompile(Tuple{typeof(Distributed.set_worker_state), Distributed.LocalProcess, Distributed.WorkerState}) +precompile(Tuple{getfield(Distributed, Symbol("#kw##rmprocs")), Array{Any, 1}, typeof(Distributed.rmprocs), Array{Int64, 1}}) +precompile(Tuple{typeof(Distributed.interrupt), Array{Int64, 1}}) +precompile(Tuple{typeof(Distributed.flush_gc_msgs)}) +precompile(Tuple{typeof(Distributed.addprocs), Int64}) +precompile(Tuple{Type{Distributed.WorkerConfig}}) +precompile(Tuple{typeof(Distributed.launch), Distributed.LocalManager, Base.Dict{Any, Any}, Array{Distributed.WorkerConfig, 1}, Base.Condition}) +precompile(Tuple{typeof(Distributed.start_worker), Base.PipeEndpoint, String}) +precompile(Tuple{typeof(Distributed.socket_reuse_port)}) +precompile(Tuple{typeof(Distributed.flush_gc_msgs)}) +precompile(Tuple{typeof(Distributed.disable_nagle), Base.TCPServer}) +precompile(Tuple{typeof(Distributed.next_tunnel_port)}) +precompile(Tuple{typeof(Base._delete!), Base.Dict{Int64, Union{Distributed.Worker, Distributed.LocalProcess}}, Int64}) +precompile(Tuple{typeof(Distributed.send_msg_), Distributed.Worker, Distributed.MsgHeader, Distributed.JoinPGRPMsg, Bool}) +precompile(Tuple{typeof(Distributed.send_msg_now), Distributed.Worker, Distributed.MsgHeader, Distributed.JoinPGRPMsg}) +precompile(Tuple{typeof(Distributed.connect_w2w), Int64, Distributed.WorkerConfig}) +precompile(Tuple{typeof(Distributed.create_worker), Distributed.LocalManager, Distributed.WorkerConfig}) +precompile(Tuple{typeof(Distributed.setup_launched_worker), Distributed.LocalManager, Distributed.WorkerConfig, Array{Int64, 1}}) +precompile(Tuple{typeof(Base.connect), Distributed.LocalManager, Int64, Distributed.WorkerConfig}) +precompile(Tuple{typeof(Distributed.read_worker_host_port), Base.Pipe}) +precompile(Tuple{typeof(Distributed.parse_connection_info), String}) +precompile(Tuple{typeof(Distributed.connect_to_worker), Base.SubString{String}, Int16}) +precompile(Tuple{typeof(Distributed.process_messages), Base.TCPSocket, Base.TCPSocket, Bool}) +precompile(Tuple{getfield(Core, Symbol("#kw#Type")), Array{Any, 1}, Type{Distributed.Worker}, Int64, Base.TCPSocket, Base.TCPSocket, Distributed.LocalManager}) +precompile(Tuple{typeof(Distributed.worker_id_from_socket), Base.TCPSocket}) +precompile(Tuple{Type{Distributed.ClusterSerializer{Base.TCPSocket}}, Base.TCPSocket}) +precompile(Tuple{typeof(Distributed.send_msg_), Distributed.Worker, Distributed.MsgHeader, Distributed.ResultMsg, Bool}) +precompile(Tuple{typeof(Distributed.send_msg_now), Distributed.Worker, Distributed.MsgHeader, Distributed.ResultMsg}) +precompile(Tuple{Type{Core.Inference.Generator{I, F} where F where I}, Type{Core.Inference.Const}, Tuple{Int64, typeof(Distributed.rmprocs)}}) +precompile(Tuple{Type{Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{Core.Inference.Const}}}, Type{Core.Inference.Const}, Tuple{Int64, typeof(Distributed.rmprocs)}}) +precompile(Tuple{typeof(Core.Inference.convert), Type{Tuple{Int64, typeof(Distributed.rmprocs)}}, Tuple{Int64, typeof(Distributed.rmprocs)}}) +precompile(Tuple{typeof(Core.Inference.collect), Type{Any}, Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{Core.Inference.Const}}}) +precompile(Tuple{typeof(Core.Inference.iteratorsize), Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{Core.Inference.Const}}}) +precompile(Tuple{typeof(Core.Inference.iteratorsize), Type{Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{Core.Inference.Const}}}}) +precompile(Tuple{typeof(Core.Inference.iteratorsize), Type{Tuple{Int64, typeof(Distributed.rmprocs)}}}) +precompile(Tuple{typeof(Core.Inference._collect), Type{Any}, Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{Core.Inference.Const}}, Core.Inference.HasLength}) +precompile(Tuple{typeof(Core.Inference.length), Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{Core.Inference.Const}}}) +precompile(Tuple{typeof(Core.Inference.length), Tuple{Int64, typeof(Distributed.rmprocs)}}) +precompile(Tuple{typeof(Core.Inference.copy!), Array{Any, 1}, Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{Core.Inference.Const}}}) +precompile(Tuple{typeof(Core.Inference.start), Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{Core.Inference.Const}}}) +precompile(Tuple{typeof(Core.Inference.start), Tuple{Int64, typeof(Distributed.rmprocs)}}) +precompile(Tuple{typeof(Core.Inference.done), Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{Core.Inference.Const}}, Int64}) +precompile(Tuple{typeof(Core.Inference.done), Tuple{Int64, typeof(Distributed.rmprocs)}, Int64}) +precompile(Tuple{typeof(Core.Inference.next), Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{Core.Inference.Const}}, Int64}) +precompile(Tuple{typeof(Core.Inference.next), Tuple{Int64, typeof(Distributed.rmprocs)}, Int64}) +precompile(Tuple{typeof(Core.Inference.getindex), Tuple{Int64, typeof(Distributed.rmprocs)}, Int64}) +precompile(Tuple{typeof(Core.Inference.start), Tuple{typeof(Distributed.rmprocs), Int64}}) +precompile(Tuple{typeof(Core.Inference.indexed_next), Tuple{typeof(Distributed.rmprocs), Int64}, Int64, Int64}) +precompile(Tuple{typeof(Core.Inference.getindex), Tuple{typeof(Distributed.rmprocs), Int64}, Int64}) +precompile(Tuple{typeof(Distributed.register_worker_streams), Distributed.Worker}) +precompile(Tuple{typeof(Distributed.register_worker_streams), Distributed.LocalProcess}) +precompile(Tuple{Type{Distributed.ClusterSerializer{Base.TCPSocket}}, Base.TCPSocket}) +precompile(Tuple{typeof(Distributed.worker_id_from_socket), Base.TCPSocket}) +precompile(Tuple{typeof(Base.convert), Type{Distributed.ClusterSerializer{I} where I<:IO}, Distributed.ClusterSerializer{Base.TCPSocket}}) +precompile(Tuple{typeof(Base.convert), Type{Distributed.ClusterManager}, Distributed.LocalManager}) +precompile(Tuple{typeof(Base.convert), Type{Distributed.WorkerConfig}, Distributed.WorkerConfig}) +precompile(Tuple{typeof(Base.get), Base.Dict{Any, Any}, Distributed.RRID, Bool}) +precompile(Tuple{typeof(Base.ht_keyindex), Base.Dict{Any, Any}, Distributed.RRID}) +precompile(Tuple{typeof(Core.Inference.isbits), Tuple{Int64, typeof(Distributed.rmprocs)}}) +precompile(Tuple{Type{Core.Inference.Generator{I, F} where F where I}, Type{QuoteNode}, Tuple{Int64, typeof(Distributed.rmprocs)}}) +precompile(Tuple{Type{Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{QuoteNode}}}, Type{QuoteNode}, Tuple{Int64, typeof(Distributed.rmprocs)}}) +precompile(Tuple{typeof(Core.Inference.iteratorsize), Type{Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{QuoteNode}}}}) +precompile(Tuple{typeof(Core.Inference._collect), Type{Any}, Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{QuoteNode}}, Core.Inference.HasLength}) +precompile(Tuple{typeof(Core.Inference.length), Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{QuoteNode}}}) +precompile(Tuple{typeof(Core.Inference.copy!), Array{Any, 1}, Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{QuoteNode}}}) +precompile(Tuple{typeof(Core.Inference.start), Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{QuoteNode}}}) +precompile(Tuple{typeof(Core.Inference.done), Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{QuoteNode}}, Int64}) +precompile(Tuple{typeof(Core.Inference.next), Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{QuoteNode}}, Int64}) +precompile(Tuple{typeof(Distributed.local_remotecall_thunk), typeof(Distributed.rmprocs), Tuple{Int64}, Array{Any, 1}}) +precompile(Tuple{typeof(Distributed.remote_do), typeof(Distributed.rmprocs), Distributed.Worker, Int64}) +precompile(Tuple{typeof(Distributed.remote_do), typeof(Distributed.rmprocs), Distributed.LocalProcess, Int64}) +precompile(Tuple{getfield(Distributed, Symbol("#kw##remote_do")), Array{Any, 1}, typeof(Distributed.remote_do), typeof(Distributed.rmprocs), Distributed.Worker, Int64}) +precompile(Tuple{getfield(Distributed, Symbol("#kw##remote_do")), Array{Any, 1}, typeof(Distributed.remote_do), typeof(Distributed.rmprocs), Distributed.LocalProcess, Int64}) +precompile(Tuple{Type{Distributed.ResultMsg}, Distributed.RemoteException}) +precompile(Tuple{Type{Distributed.ResultMsg}, Symbol}) +precompile(Tuple{typeof(Distributed.send_msg_now), Base.TCPSocket, Distributed.MsgHeader, Distributed.ResultMsg}) +precompile(Tuple{typeof(Base._delete!), Base.Dict{Int64, Union{Distributed.Worker, Distributed.LocalProcess}}, Int64}) +precompile(Tuple{typeof(Distributed.def_rv_channel)}) +precompile(Tuple{typeof(Base.setindex!), Base.Dict{Any, Any}, Distributed.RemoteValue, Distributed.RRID}) +precompile(Tuple{typeof(Base.ht_keyindex2!), Base.Dict{Any, Any}, Distributed.RRID}) +precompile(Tuple{typeof(Base._setindex!), Base.Dict{Any, Any}, Distributed.RemoteValue, Distributed.RRID, Int64}) +precompile(Tuple{typeof(Base.notify), Base.Condition, Distributed.ProcessExitedException, Bool, Bool}) +precompile(Tuple{typeof(Distributed.process_messages), Base.TCPSocket, Base.TCPSocket, Bool}) +precompile(Tuple{typeof(Base.pop!), Base.Dict{Int64, Union{Distributed.Worker, Distributed.LocalProcess}}, Int64, Void}) +precompile(Tuple{typeof(Distributed.send_connection_hdr), Distributed.Worker, Bool}) +precompile(Tuple{typeof(Distributed.deregister_worker), Distributed.ProcessGroup, Int64}) +precompile(Tuple{typeof(Distributed.process_hdr), Base.TCPSocket, Bool}) +precompile(Tuple{typeof(Distributed.deserialize_msg), Distributed.ClusterSerializer{Base.TCPSocket}}) +precompile(Tuple{typeof(Distributed.null_id), Distributed.RRID}) +precompile(Tuple{typeof(Distributed.deliver_result), Base.TCPSocket, Symbol, Distributed.RRID, Distributed.RemoteException}) +precompile(Tuple{typeof(Distributed.disable_nagle), Base.TCPSocket}) +precompile(Tuple{typeof(Distributed.message_handler_loop), Base.TCPSocket, Base.TCPSocket, Bool}) +precompile(Tuple{typeof(Distributed.process_tcp_streams), Base.TCPSocket, Base.TCPSocket, Bool}) +precompile(Tuple{Type{Distributed.JoinPGRPMsg}, Int64, Array{Union{Tuple{Any, Int64}, Tuple{Tuple{}, Any, Bool}}, 1}, Symbol, Bool}) +precompile(Tuple{typeof(Base.Serializer.serialize), Distributed.ClusterSerializer{Base.TCPSocket}, Distributed.JoinPGRPMsg}) +precompile(Tuple{typeof(Base.Serializer.serialize), Distributed.ClusterSerializer{Base.TCPSocket}, Int64}) +precompile(Tuple{typeof(Base.Serializer.serialize), Distributed.ClusterSerializer{Base.TCPSocket}, Array{Union{Tuple{Any, Int64}, Tuple{Tuple{}, Any, Bool}}, 1}}) +precompile(Tuple{typeof(Base.Serializer.should_send_whole_type), Distributed.ClusterSerializer{Base.TCPSocket}, Type{Any}}) +precompile(Tuple{typeof(Base.Serializer.serialize), Distributed.ClusterSerializer{Base.TCPSocket}, SimpleVector}) +precompile(Tuple{typeof(Base.Serializer.serialize_type_data), Distributed.ClusterSerializer{Base.TCPSocket}, Type{Any}, Bool}) +precompile(Tuple{typeof(Base.Serializer.serialize), Distributed.ClusterSerializer{Base.TCPSocket}, Type{Any}}) +precompile(Tuple{typeof(Base.Serializer.serialize), Distributed.ClusterSerializer{Base.TCPSocket}, Tuple{Int64}}) +precompile(Tuple{typeof(Base.Serializer.serialize_cycle), Distributed.ClusterSerializer{Base.TCPSocket}, Type{Union{Tuple{Any, Int64}, Tuple{Tuple{}, Any, Bool}}}}) +precompile(Tuple{typeof(Base.Serializer.serialize_type), Distributed.ClusterSerializer{Base.TCPSocket}, DataType}) +precompile(Tuple{typeof(Base.Serializer.serialize), Distributed.ClusterSerializer{Base.TCPSocket}, Bool}) +precompile(Tuple{typeof(Distributed.serialize_global_from_main), Distributed.ClusterSerializer{Base.TCPSocket}, Symbol}) +precompile(Tuple{typeof(Base.Serializer.serialize_mod_names), Distributed.ClusterSerializer{Base.TCPSocket}, Module}) +precompile(Tuple{typeof(Base.Serializer.serialize), Distributed.ClusterSerializer{Base.TCPSocket}, Symbol}) +precompile(Tuple{typeof(Base.Serializer.serialize), Distributed.ClusterSerializer{Base.TCPSocket}, Module}) +precompile(Tuple{typeof(Base.Serializer.serialize), Distributed.ClusterSerializer{Base.TCPSocket}, Array{Any, 1}}) +precompile(Tuple{typeof(Base.Serializer.serialize_cycle), Distributed.ClusterSerializer{Base.TCPSocket}, TypeName}) +precompile(Tuple{typeof(Base.Serializer.serialize_typename), Distributed.ClusterSerializer{Base.TCPSocket}, TypeName}) +precompile(Tuple{typeof(Base.Serializer.serialize), Distributed.ClusterSerializer{Base.TCPSocket}, Array{Symbol, 1}}) +precompile(Tuple{typeof(Base.Serializer.serialize), Distributed.ClusterSerializer{Base.TCPSocket}, TypeName}) +precompile(Tuple{typeof(Base.Serializer.serialize_cycle_header), Distributed.ClusterSerializer{Base.TCPSocket}, Type{Union{Tuple{Any, Int64}, Tuple{Tuple{}, Any, Bool}}}}) +precompile(Tuple{typeof(Base.Serializer.serialize_any), Distributed.ClusterSerializer{Base.TCPSocket}, Type{Union{Tuple{Any, Int64}, Tuple{Tuple{}, Any, Bool}}}}) +precompile(Tuple{typeof(Distributed.send_msg_), Distributed.Worker, Distributed.MsgHeader, Distributed.ResultMsg, Bool}) +precompile(Tuple{typeof(Distributed.send_msg_now), Distributed.Worker, Distributed.MsgHeader, Distributed.ResultMsg}) +precompile(Tuple{Type{Core.Inference.Generator{I, F} where F where I}, Type{Core.Inference.Const}, Tuple{Int64, typeof(Distributed.rmprocs)}}) +precompile(Tuple{Type{Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{Core.Inference.Const}}}, Type{Core.Inference.Const}, Tuple{Int64, typeof(Distributed.rmprocs)}}) +precompile(Tuple{typeof(Core.Inference.convert), Type{Tuple{Int64, typeof(Distributed.rmprocs)}}, Tuple{Int64, typeof(Distributed.rmprocs)}}) +precompile(Tuple{typeof(Core.Inference.collect), Type{Any}, Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{Core.Inference.Const}}}) +precompile(Tuple{typeof(Core.Inference.iteratorsize), Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{Core.Inference.Const}}}) +precompile(Tuple{typeof(Core.Inference.iteratorsize), Type{Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{Core.Inference.Const}}}}) +precompile(Tuple{typeof(Core.Inference.iteratorsize), Type{Tuple{Int64, typeof(Distributed.rmprocs)}}}) +precompile(Tuple{typeof(Core.Inference._collect), Type{Any}, Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{Core.Inference.Const}}, Core.Inference.HasLength}) +precompile(Tuple{typeof(Core.Inference.length), Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{Core.Inference.Const}}}) +precompile(Tuple{typeof(Core.Inference.length), Tuple{Int64, typeof(Distributed.rmprocs)}}) +precompile(Tuple{typeof(Core.Inference.copy!), Array{Any, 1}, Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{Core.Inference.Const}}}) +precompile(Tuple{typeof(Core.Inference.start), Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{Core.Inference.Const}}}) +precompile(Tuple{typeof(Core.Inference.start), Tuple{Int64, typeof(Distributed.rmprocs)}}) +precompile(Tuple{typeof(Core.Inference.done), Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{Core.Inference.Const}}, Int64}) +precompile(Tuple{typeof(Core.Inference.done), Tuple{Int64, typeof(Distributed.rmprocs)}, Int64}) +precompile(Tuple{typeof(Core.Inference.next), Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{Core.Inference.Const}}, Int64}) +precompile(Tuple{typeof(Core.Inference.next), Tuple{Int64, typeof(Distributed.rmprocs)}, Int64}) +precompile(Tuple{typeof(Core.Inference.getindex), Tuple{Int64, typeof(Distributed.rmprocs)}, Int64}) +precompile(Tuple{typeof(Core.Inference.start), Tuple{typeof(Distributed.rmprocs), Int64}}) +precompile(Tuple{typeof(Core.Inference.indexed_next), Tuple{typeof(Distributed.rmprocs), Int64}, Int64, Int64}) +precompile(Tuple{typeof(Core.Inference.getindex), Tuple{typeof(Distributed.rmprocs), Int64}, Int64}) +precompile(Tuple{typeof(Base.Serializer.deserialize), Distributed.ClusterSerializer{Base.TCPSocket}}) +precompile(Tuple{typeof(Base.Serializer.deserialize_cycle), Distributed.ClusterSerializer{Base.TCPSocket}, Expr}) +precompile(Tuple{typeof(Base.Serializer.handle_deserialize), Distributed.ClusterSerializer{Base.TCPSocket}, Int32}) +precompile(Tuple{typeof(Base.Serializer.deserialize_array), Distributed.ClusterSerializer{Base.TCPSocket}}) +precompile(Tuple{typeof(Base.Serializer.deserialize_datatype), Distributed.ClusterSerializer{Base.TCPSocket}}) +precompile(Tuple{typeof(Base.Serializer.deserialize_expr), Distributed.ClusterSerializer{Base.TCPSocket}, Int64}) +precompile(Tuple{typeof(Core.Inference.isbits), Tuple{Int64, typeof(Distributed.rmprocs)}}) +precompile(Tuple{Type{Core.Inference.Generator{I, F} where F where I}, Type{QuoteNode}, Tuple{Int64, typeof(Distributed.rmprocs)}}) +precompile(Tuple{Type{Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{QuoteNode}}}, Type{QuoteNode}, Tuple{Int64, typeof(Distributed.rmprocs)}}) +precompile(Tuple{typeof(Core.Inference.iteratorsize), Type{Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{QuoteNode}}}}) +precompile(Tuple{typeof(Core.Inference._collect), Type{Any}, Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{QuoteNode}}, Core.Inference.HasLength}) +precompile(Tuple{typeof(Core.Inference.length), Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{QuoteNode}}}) +precompile(Tuple{typeof(Core.Inference.copy!), Array{Any, 1}, Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{QuoteNode}}}) +precompile(Tuple{typeof(Core.Inference.start), Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{QuoteNode}}}) +precompile(Tuple{typeof(Core.Inference.done), Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{QuoteNode}}, Int64}) +precompile(Tuple{typeof(Core.Inference.next), Core.Inference.Generator{Tuple{Int64, typeof(Distributed.rmprocs)}, Type{QuoteNode}}, Int64}) +precompile(Tuple{typeof(Distributed.local_remotecall_thunk), typeof(Distributed.rmprocs), Tuple{Int64}, Array{Any, 1}}) +precompile(Tuple{typeof(Distributed.remote_do), typeof(Distributed.rmprocs), Distributed.Worker, Int64}) +precompile(Tuple{typeof(Distributed.remote_do), typeof(Distributed.rmprocs), Distributed.LocalProcess, Int64}) +precompile(Tuple{getfield(Distributed, Symbol("#kw##remote_do")), Array{Any, 1}, typeof(Distributed.remote_do), typeof(Distributed.rmprocs), Distributed.Worker, Int64}) +precompile(Tuple{getfield(Distributed, Symbol("#kw##remote_do")), Array{Any, 1}, typeof(Distributed.remote_do), typeof(Distributed.rmprocs), Distributed.LocalProcess, Int64}) +precompile(Tuple{Type{Distributed.ResultMsg}, Distributed.RemoteException}) +precompile(Tuple{Type{Distributed.ResultMsg}, Symbol}) +precompile(Tuple{typeof(Distributed.send_msg_now), Base.TCPSocket, Distributed.MsgHeader, Distributed.ResultMsg}) +precompile(Tuple{typeof(Base.notify), Base.Condition, Distributed.ProcessExitedException, Bool, Bool}) +precompile(Tuple{typeof(Base.pop!), Base.Dict{Int64, Union{Distributed.Worker, Distributed.LocalProcess}}, Int64, Void}) +precompile(Tuple{typeof(Distributed.deregister_worker), Distributed.ProcessGroup, Int64}) +precompile(Tuple{typeof(Distributed.process_hdr), Base.TCPSocket, Bool}) +precompile(Tuple{typeof(Distributed.null_id), Distributed.RRID}) +precompile(Tuple{typeof(Distributed.deliver_result), Base.TCPSocket, Symbol, Distributed.RRID, Distributed.RemoteException}) +precompile(Tuple{typeof(Distributed.disable_nagle), Base.TCPSocket}) +precompile(Tuple{typeof(Distributed.message_handler_loop), Base.TCPSocket, Base.TCPSocket, Bool}) +precompile(Tuple{typeof(Distributed.process_tcp_streams), Base.TCPSocket, Base.TCPSocket, Bool}) +precompile(Tuple{typeof(Base.Serializer.deserialize), Distributed.ClusterSerializer{Base.TCPSocket}, Type{Union}}) +precompile(Tuple{typeof(Base.Serializer.deserialize), Distributed.ClusterSerializer{Base.TCPSocket}, Type{Module}}) +precompile(Tuple{typeof(Base.Serializer.deserialize), Distributed.ClusterSerializer{Base.TCPSocket}, Type{SimpleVector}}) +precompile(Tuple{Type{Distributed.JoinPGRPMsg}, Int64, Array{Union{Tuple{Any, Int64}, Tuple{Tuple{}, Any, Bool}}, 1}, Symbol, Bool}) +precompile(Tuple{typeof(Distributed.handle_msg), Distributed.JoinPGRPMsg, Distributed.MsgHeader, Base.TCPSocket, Base.TCPSocket, Base.VersionNumber}) +precompile(Tuple{Type{Distributed.WorkerConfig}}) +precompile(Tuple{typeof(Distributed.send_msg_), Distributed.Worker, Distributed.MsgHeader, Distributed.JoinCompleteMsg, Bool}) +precompile(Tuple{typeof(Distributed.send_msg_now), Distributed.Worker, Distributed.MsgHeader, Distributed.JoinCompleteMsg}) +precompile(Tuple{getfield(Core, Symbol("#kw#Type")), Array{Any, 1}, Type{Distributed.Worker}, Int64, Base.TCPSocket, Base.TCPSocket, Distributed.DefaultClusterManager}) +precompile(Tuple{typeof(Distributed.register_worker_streams), Distributed.Worker}) +precompile(Tuple{typeof(Distributed.register_worker_streams), Distributed.LocalProcess}) +precompile(Tuple{typeof(Base.convert), Type{Distributed.ClusterSerializer{I} where I<:IO}, Distributed.ClusterSerializer{Base.TCPSocket}}) +precompile(Tuple{typeof(Base.convert), Type{Distributed.ClusterManager}, Distributed.DefaultClusterManager}) +precompile(Tuple{typeof(Base.convert), Type{Distributed.WorkerConfig}, Distributed.WorkerConfig}) +precompile(Tuple{typeof(Distributed.send_connection_hdr), Distributed.Worker, Bool}) +precompile(Tuple{typeof(Distributed.manage), Distributed.LocalManager, Int64, Distributed.WorkerConfig, Symbol}) +precompile(Tuple{typeof(Base.Serializer.serialize), Distributed.ClusterSerializer{Base.TCPSocket}, Distributed.JoinCompleteMsg}) +precompile(Tuple{typeof(Base.Serializer.serialize), Distributed.ClusterSerializer{Base.TCPSocket}, Int64}) +precompile(Tuple{typeof(Base.Serializer.deserialize), Distributed.ClusterSerializer{Base.TCPSocket}}) +precompile(Tuple{typeof(Base.Serializer.deserialize_cycle), Distributed.ClusterSerializer{Base.TCPSocket}, Expr}) +precompile(Tuple{typeof(Base.Serializer.handle_deserialize), Distributed.ClusterSerializer{Base.TCPSocket}, Int32}) +precompile(Tuple{typeof(Base.Serializer.deserialize_array), Distributed.ClusterSerializer{Base.TCPSocket}}) +precompile(Tuple{typeof(Base.Serializer.deserialize_datatype), Distributed.ClusterSerializer{Base.TCPSocket}}) +precompile(Tuple{typeof(Base.Serializer.deserialize_expr), Distributed.ClusterSerializer{Base.TCPSocket}, Int64}) +precompile(Tuple{typeof(Base.Serializer.deserialize), Distributed.ClusterSerializer{Base.TCPSocket}, Type{Int64}}) +precompile(Tuple{Type{Distributed.JoinCompleteMsg}, Int64, Int64}) +precompile(Tuple{typeof(Distributed.handle_msg), Distributed.JoinCompleteMsg, Distributed.MsgHeader, Base.TCPSocket, Base.TCPSocket, Base.VersionNumber}) +precompile(Tuple{typeof(Base.hash), Distributed.RemoteChannel{Base.Channel{Any}}, UInt64}) +precompile(Tuple{typeof(Base.ht_keyindex), Base.Dict{WeakRef, Void}, Distributed.RemoteChannel{Base.Channel{Any}}}) +precompile(Tuple{typeof(Distributed.remotecall_fetch), typeof(Distributed.put_ref), Distributed.Worker, Distributed.RRID, Distributed.WorkerPool}) +precompile(Tuple{typeof(Distributed.remotecall_fetch), typeof(Distributed.put_ref), Distributed.LocalProcess, Distributed.RRID, Distributed.WorkerPool}) +precompile(Tuple{getfield(Distributed, Symbol("#kw##remotecall_fetch")), Array{Any, 1}, typeof(Distributed.remotecall_fetch), typeof(Distributed.put_ref), Distributed.Worker, Distributed.RRID, Distributed.WorkerPool}) +precompile(Tuple{getfield(Distributed, Symbol("#kw##remotecall_fetch")), Array{Any, 1}, typeof(Distributed.remotecall_fetch), typeof(Distributed.put_ref), Distributed.LocalProcess, Distributed.RRID, Distributed.WorkerPool}) +precompile(Tuple{typeof(Base.finalizer), Distributed.RemoteChannel{Base.Channel{Any}}, typeof(Distributed.finalize_ref)}) +precompile(Tuple{typeof(Distributed.test_existing_ref), Distributed.RemoteChannel{Base.Channel{Any}}}) +precompile(Tuple{Type{Distributed.RemoteChannel{T} where T<:Base.AbstractChannel}, Int64}) +precompile(Tuple{Type{Distributed.WorkerPool}}) +precompile(Tuple{typeof(Distributed.default_worker_pool)}) +precompile(Tuple{typeof(Distributed.call_on_owner), typeof(Distributed.put_ref), Distributed.RemoteChannel{Base.Channel{Any}}, Distributed.WorkerPool}) +precompile(Tuple{typeof(Distributed.put_ref), Distributed.RRID, Distributed.WorkerPool}) +precompile(Tuple{typeof(Base.put!), Distributed.RemoteValue, Distributed.WorkerPool}) +precompile(Tuple{typeof(Base.put!), Base.Channel{Any}, Distributed.WorkerPool}) +precompile(Tuple{typeof(Base.put_buffered), Base.Channel{Any}, Distributed.WorkerPool}) +precompile(Tuple{typeof(Base.put_unbuffered), Base.Channel{Any}, Distributed.WorkerPool}) +precompile(Tuple{typeof(Base.push!), Distributed.WorkerPool, Int64}) +precompile(Tuple{typeof(Base.Serializer.serialize), Distributed.ClusterSerializer{Base.TCPSocket}, Distributed.RemoteDoMsg}) +precompile(Tuple{typeof(Base.Serializer.serialize), Distributed.ClusterSerializer{Base.TCPSocket}, typeof(Distributed.set_valid_processes)}) +precompile(Tuple{typeof(Base.Serializer.serialize), Distributed.ClusterSerializer{Base.TCPSocket}, Tuple{Array{Int64, 1}}}) +precompile(Tuple{typeof(Base.Serializer.serialize), Distributed.ClusterSerializer{Base.TCPSocket}, Array{Int64, 1}}) +precompile(Tuple{typeof(Base.Serializer.deserialize), Distributed.ClusterSerializer{Base.TCPSocket}, Type{typeof(Distributed.set_valid_processes)}}) +precompile(Tuple{typeof(Distributed.handle_msg), Distributed.RemoteDoMsg, Distributed.MsgHeader, Base.TCPSocket, Base.TCPSocket, Base.VersionNumber}) +precompile(Tuple{typeof(Distributed.set_valid_processes), Array{Int64, 1}}) +precompile(Tuple{typeof(Distributed._rmprocs), Array{Int64, 1}, Float64}) +precompile(Tuple{typeof(Base.kill), Distributed.LocalManager, Int64, Distributed.WorkerConfig}) +precompile(Tuple{typeof(Base.Serializer.serialize), Distributed.ClusterSerializer{Base.TCPSocket}, typeof(Base.exit)}) +precompile(Tuple{typeof(Base.Serializer.serialize), Distributed.ClusterSerializer{Base.TCPSocket}, Tuple{}}) +precompile(Tuple{typeof(Base.Serializer.deserialize), Distributed.ClusterSerializer{Base.TCPSocket}, Type{typeof(Base.exit)}}) +precompile(Tuple{typeof(Distributed.finalize_ref), Distributed.RemoteChannel{Base.Channel{Any}}}) +precompile(Tuple{typeof(Distributed.send_del_client), Distributed.RemoteChannel{Base.Channel{Any}}}) +precompile(Tuple{typeof(Base.:(==)), Distributed.RemoteChannel{Base.Channel{Any}}, Distributed.RemoteChannel{Base.Channel{Any}}}) +precompile(Tuple{getfield(Distributed, Symbol("#kw##remotecall_fetch")), Array{Any, 1}, typeof(Distributed.remotecall_fetch), typeof(Base.open), Distributed.LocalProcess, typeof(Base.read), String}) +precompile(Tuple{getfield(Distributed, Symbol("#kw##remotecall_fetch")), Array{Any, 1}, typeof(Distributed.remotecall_fetch), typeof(Base.open), Distributed.Worker, typeof(Base.read), String}) +precompile(Tuple{getfield(Distributed, Symbol("#kw##remote_do")), Array{Any, 1}, typeof(Distributed.remote_do), typeof(Base.exit), Distributed.Worker}) +precompile(Tuple{getfield(Distributed, Symbol("#kw##rmprocs")), Array{Any, 1}, typeof(Distributed.rmprocs), Array{Int64, 1}}) +precompile(Tuple{Type{Distributed.Future}, Int64}) +precompile(Tuple{typeof(Distributed.flush_gc_msgs), Distributed.Worker}) +precompile(Tuple{typeof(Distributed.remote_do), typeof(Base.exit), Distributed.Worker}) +precompile(Tuple{typeof(Distributed.send_del_client), Distributed.Future}) +precompile(Tuple{typeof(Distributed.send_msg), Distributed.Worker, Distributed.MsgHeader, Distributed.CallMsg{:call}}) +precompile(Tuple{typeof(Distributed.send_msg_), Distributed.Worker, Distributed.MsgHeader, Distributed.CallMsg{:call}, Bool}) +precompile(Tuple{typeof(Distributed.send_msg), Distributed.Worker, Distributed.MsgHeader, Distributed.CallMsg{:call_fetch}}) +precompile(Tuple{typeof(Distributed.send_msg_), Distributed.Worker, Distributed.MsgHeader, Distributed.CallMsg{:call_fetch}, Bool}) +precompile(Tuple{typeof(Distributed.send_msg), Distributed.Worker, Distributed.MsgHeader, Distributed.RemoteDoMsg}) +precompile(Tuple{typeof(Distributed.send_msg_), Distributed.Worker, Distributed.MsgHeader, Distributed.RemoteDoMsg, Bool}) +precompile(Tuple{typeof(Distributed.terminate_all_workers)}) +precompile(Tuple{typeof(Distributed.test_existing_ref), Distributed.Future}) +precompile(Tuple{typeof(Base.finalizer), Distributed.Future, typeof(Distributed.finalize_ref)}) +precompile(Tuple{typeof(Base.hash), Distributed.Future, UInt64}) +precompile(Tuple{typeof(Base.ht_keyindex), Base.Dict{WeakRef, Void}, Distributed.Future}) +precompile(Tuple{typeof(Base.sync_add), Distributed.Future}) +precompile(Tuple{Type{Union{}}, Distributed.RRID}) diff --git a/base/distributed/process_messages.jl b/stdlib/Distributed/src/process_messages.jl similarity index 100% rename from base/distributed/process_messages.jl rename to stdlib/Distributed/src/process_messages.jl diff --git a/base/distributed/remotecall.jl b/stdlib/Distributed/src/remotecall.jl similarity index 100% rename from base/distributed/remotecall.jl rename to stdlib/Distributed/src/remotecall.jl diff --git a/base/distributed/workerpool.jl b/stdlib/Distributed/src/workerpool.jl similarity index 100% rename from base/distributed/workerpool.jl rename to stdlib/Distributed/src/workerpool.jl diff --git a/test/distributed_exec.jl b/stdlib/Distributed/test/distributed_exec.jl similarity index 88% rename from test/distributed_exec.jl rename to stdlib/Distributed/test/distributed_exec.jl index a68781ec01285c..5815d38ea35aa9 100644 --- a/test/distributed_exec.jl +++ b/stdlib/Distributed/test/distributed_exec.jl @@ -1,7 +1,9 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -using Test -include("testenv.jl") +using Test, Distributed +import Distributed: launch, manage + +include(joinpath(JULIA_HOME, "..", "share", "julia", "test", "testenv.jl")) # Test a few "remote" invocations when no workers are present @test remote(myid)() == 1 @@ -20,7 +22,7 @@ id_other = filter(x -> x != id_me, procs())[rand(1:(nprocs()-1))] # Test remote() let - pool = Base.default_worker_pool() + pool = default_worker_pool() count = 0 count_condition = Condition() @@ -98,25 +100,25 @@ testf(id_other) # Distributed GC tests for Futures function test_futures_dgc(id) f = remotecall(myid, id) - fid = Base.remoteref_id(f) + fid = remoteref_id(f) # remote value should be deleted after a fetch - @test remotecall_fetch(k->(yield();haskey(Base.Distributed.PGRP.refs, k)), id, fid) == true + @test remotecall_fetch(k->(yield();haskey(Distributed.PGRP.refs, k)), id, fid) == true @test isnull(f.v) == true @test fetch(f) == id @test isnull(f.v) == false yield(); # flush gc msgs - @test remotecall_fetch(k->(yield();haskey(Base.Distributed.PGRP.refs, k)), id, fid) == false + @test remotecall_fetch(k->(yield();haskey(Distributed.PGRP.refs, k)), id, fid) == false # if unfetched, it should be deleted after a finalize f = remotecall(myid, id) - fid = Base.remoteref_id(f) - @test remotecall_fetch(k->(yield();haskey(Base.Distributed.PGRP.refs, k)), id, fid) == true + fid = remoteref_id(f) + @test remotecall_fetch(k->(yield();haskey(Distributed.PGRP.refs, k)), id, fid) == true @test isnull(f.v) == true finalize(f) yield(); # flush gc msgs - @test remotecall_fetch(k->(yield();haskey(Base.Distributed.PGRP.refs, k)), id, fid) == false + @test remotecall_fetch(k->(yield();haskey(Distributed.PGRP.refs, k)), id, fid) == false end test_futures_dgc(id_me) @@ -126,40 +128,40 @@ test_futures_dgc(id_other) wid1 = workers()[1] wid2 = workers()[2] f = remotecall(myid, wid1) -fid = Base.remoteref_id(f) +fid = remoteref_id(f) fstore = RemoteChannel(wid2) put!(fstore, f) @test fetch(f) == wid1 -@test remotecall_fetch(k->haskey(Base.Distributed.PGRP.refs, k), wid1, fid) == true +@test remotecall_fetch(k->haskey(Distributed.PGRP.refs, k), wid1, fid) == true remotecall_fetch(r->(fetch(fetch(r)); yield()), wid2, fstore) sleep(0.5) # to ensure that wid2 gc messages have been executed on wid1 -@test remotecall_fetch(k->haskey(Base.Distributed.PGRP.refs, k), wid1, fid) == false +@test remotecall_fetch(k->haskey(Distributed.PGRP.refs, k), wid1, fid) == false # put! should release remote reference since it would have been cached locally f = Future(wid1) -fid = Base.remoteref_id(f) +fid = remoteref_id(f) # should not be created remotely till accessed -@test remotecall_fetch(k->haskey(Base.Distributed.PGRP.refs, k), wid1, fid) == false +@test remotecall_fetch(k->haskey(Distributed.PGRP.refs, k), wid1, fid) == false # create it remotely isready(f) -@test remotecall_fetch(k->haskey(Base.Distributed.PGRP.refs, k), wid1, fid) == true +@test remotecall_fetch(k->haskey(Distributed.PGRP.refs, k), wid1, fid) == true put!(f, :OK) -@test remotecall_fetch(k->haskey(Base.Distributed.PGRP.refs, k), wid1, fid) == false +@test remotecall_fetch(k->haskey(Distributed.PGRP.refs, k), wid1, fid) == false @test fetch(f) == :OK # RemoteException should be thrown on a put! when another process has set the value f = Future(wid1) -fid = Base.remoteref_id(f) +fid = remoteref_id(f) fstore = RemoteChannel(wid2) put!(fstore, f) # send f to wid2 put!(f, :OK) # set value from master -@test remotecall_fetch(k->haskey(Base.Distributed.PGRP.refs, k), wid1, fid) == true +@test remotecall_fetch(k->haskey(Distributed.PGRP.refs, k), wid1, fid) == true testval = remotecall_fetch(wid2, fstore) do x try @@ -179,15 +181,15 @@ end function test_remoteref_dgc(id) rr = RemoteChannel(id) put!(rr, :OK) - rrid = Base.remoteref_id(rr) + rrid = remoteref_id(rr) # remote value should be deleted after finalizing the ref - @test remotecall_fetch(k->(yield();haskey(Base.Distributed.PGRP.refs, k)), id, rrid) == true + @test remotecall_fetch(k->(yield();haskey(Distributed.PGRP.refs, k)), id, rrid) == true @test fetch(rr) == :OK - @test remotecall_fetch(k->(yield();haskey(Base.Distributed.PGRP.refs, k)), id, rrid) == true + @test remotecall_fetch(k->(yield();haskey(Distributed.PGRP.refs, k)), id, rrid) == true finalize(rr) yield(); # flush gc msgs - @test remotecall_fetch(k->(yield();haskey(Base.Distributed.PGRP.refs, k)), id, rrid) == false + @test remotecall_fetch(k->(yield();haskey(Distributed.PGRP.refs, k)), id, rrid) == false end test_remoteref_dgc(id_me) test_remoteref_dgc(id_other) @@ -196,17 +198,17 @@ test_remoteref_dgc(id_other) let wid1 = workers()[1], wid2 = workers()[2], rr = RemoteChannel(wid1), - rrid = Base.remoteref_id(rr), + rrid = remoteref_id(rr), fstore = RemoteChannel(wid2) put!(fstore, rr) - @test remotecall_fetch(k -> haskey(Base.Distributed.PGRP.refs, k), wid1, rrid) == true + @test remotecall_fetch(k -> haskey(Distributed.PGRP.refs, k), wid1, rrid) == true finalize(rr) # finalize locally yield() # flush gc msgs - @test remotecall_fetch(k -> haskey(Base.Distributed.PGRP.refs, k), wid1, rrid) == true + @test remotecall_fetch(k -> haskey(Distributed.PGRP.refs, k), wid1, rrid) == true remotecall_fetch(r -> (finalize(take!(r)); yield(); nothing), wid2, fstore) # finalize remotely sleep(0.5) # to ensure that wid2 messages have been executed on wid1 - @test remotecall_fetch(k -> haskey(Base.Distributed.PGRP.refs, k), wid1, rrid) == false + @test remotecall_fetch(k -> haskey(Distributed.PGRP.refs, k), wid1, rrid) == false end # Tests for issue #23109 - should not hang. @@ -249,7 +251,7 @@ test_indexing(RemoteChannel()) test_indexing(RemoteChannel(id_other)) # Test ser/deser to non-ClusterSerializer objects. -function test_regular_io_ser(ref::Base.Distributed.AbstractRemoteRef) +function test_regular_io_ser(ref::Distributed.AbstractRemoteRef) io = IOBuffer() serialize(io, ref) seekstart(io) @@ -522,6 +524,16 @@ end # Start test for various kw arg combinations walk_args(1) +include(joinpath(JULIA_HOME, "..", "share", "julia", "test", "generic_map_tests.jl")) +empty_pool = WorkerPool([myid()]) +pmap_fallback = (f, c...) -> pmap(empty_pool, f, c...) +generic_map_tests(pmap_fallback) + +# pmap with various types. Test for equivalence with map +run_map_equivalence_tests(pmap) +@test pmap(uppercase, "Hello World!") == map(uppercase, "Hello World!") + + # Simple test for pmap throws error let error_thrown = false try @@ -540,7 +552,7 @@ end n = 10 as = [rand(4,4) for i in 1:n] bs = deepcopy(as) -cs = collect(Base.Distributed.pgenerate(x->(sleep(rand()*0.1); svdfact(x)), bs)) +cs = collect(Distributed.pgenerate(x->(sleep(rand()*0.1); svdfact(x)), bs)) svdas = map(svdfact, as) for i in 1:n @test cs[i][:U] ≈ svdas[i][:U] @@ -548,94 +560,6 @@ for i in 1:n @test cs[i][:V] ≈ svdas[i][:V] end -# Test asyncmap -@test allunique(asyncmap(x->(sleep(1.0);object_id(current_task())), 1:10)) - -# num tasks -@test length(unique(asyncmap(x->(yield();object_id(current_task())), 1:20; ntasks=5))) == 5 - -# default num tasks -@test length(unique(asyncmap(x->(yield();object_id(current_task())), 1:200))) == 100 - -# ntasks as a function -let nt=0 - global nt_func - nt_func() = (v=div(nt, 25); nt+=1; v) # increment number of tasks by 1 for every 25th call. - # nt_func() will be called initally once and then for every - # iteration -end -@test length(unique(asyncmap(x->(yield();object_id(current_task())), 1:200; ntasks=nt_func))) == 7 - -# batch mode tests -let ctr=0 - global next_ctr - next_ctr() = (ctr+=1; ctr) -end -resp = asyncmap(x->(v=next_ctr(); map(_->v, x)), 1:22; ntasks=5, batch_size=5) -@test length(resp) == 22 -@test length(unique(resp)) == 5 - -input = rand(1:1000, 100) -@test asyncmap(x->map(args->identity(args...), x), input; ntasks=5, batch_size=5) == input - -# check whether shape is retained -a=rand(2,2) -b=asyncmap(identity, a) -@test a == b -@test size(a) == size(b) - -# check with an iterator that does not implement size() -c=Channel(32); foreach(i->put!(c,i), 1:10); close(c) -b=asyncmap(identity, c) -@test Int[1:10...] == b -@test size(b) == (10,) - -# check with an iterator that has only implements length() -len_only_iterable = (1,2,3,4,5) -@test Base.iteratorsize(len_only_iterable) == Base.HasLength() -@test asyncmap(identity, len_only_iterable) == map(identity, len_only_iterable) - -# Error conditions -@test_throws ArgumentError asyncmap(identity, 1:10; batch_size=0) -@test_throws ArgumentError asyncmap(identity, 1:10; batch_size="10") -@test_throws ArgumentError asyncmap(identity, 1:10; ntasks="10") - -# asyncmap and pmap with various types. Test for equivalence with map -function testmap_equivalence(f, c...) - x1 = asyncmap(f,c...) - x2 = pmap(f,c...) - x3 = map(f,c...) - - if Base.iteratorsize == Base.HasShape() - @test size(x1) == size(x3) - @test size(x2) == size(x3) - else - @test length(x1) == length(x3) - @test length(x2) == length(x3) - end - - @test eltype(x1) == eltype(x3) - @test eltype(x2) == eltype(x3) - - for (v1,v2) in zip(x1,x3) - @test v1==v2 - end - for (v1,v2) in zip(x2,x3) - @test v1==v2 - end -end - -testmap_equivalence(identity, (1,2,3,4)) -testmap_equivalence(x->x>0 ? 1.0 : 0.0, sparse(1.0I, 5, 5)) -testmap_equivalence((x,y,z)->x+y+z, 1,2,3) -testmap_equivalence(x->x ? false : true, BitArray(10,10)) -testmap_equivalence(x->"foobar", BitArray(10,10)) -testmap_equivalence((x,y,z)->string(x,y,z), BitArray(10), ones(10), "1234567890") - -@test asyncmap(uppercase, "Hello World!") == map(uppercase, "Hello World!") -@test pmap(uppercase, "Hello World!") == map(uppercase, "Hello World!") - - # Test that the default worker pool cycles through all workers pmap(_->myid(), 1:nworkers()) # priming run @test nworkers() == length(unique(pmap(_->myid(), 1:100))) @@ -677,8 +601,8 @@ if DoFullTest all_w = workers() # Test sending fake data to workers. The worker processes will print an # error message but should not terminate. - for w in Base.Distributed.PGRP.workers - if isa(w, Base.Distributed.Worker) + for w in Distributed.PGRP.workers + if isa(w, Distributed.Worker) local s = connect(get(w.config.host), get(w.config.port)) write(s, randstring(32)) end @@ -823,7 +747,7 @@ function test_f_args(result, args...; kwargs...) remote_do(args...; kwargs...) end -for tid in [id_other, id_me, Base.default_worker_pool()] +for tid in [id_other, id_me, default_worker_pool()] test_f_args(1, f_args, tid, 1) test_f_args(3, f_args, tid, 1, 2) test_f_args(5, f_args, tid, 1; kw1=4) @@ -935,6 +859,7 @@ end # Test calling @everywhere from a module not defined on the workers module LocalBar + using Distributed bar() = @everywhere new_bar()=myid() end LocalBar.bar() @@ -1015,7 +940,7 @@ function get_remote_num_threads(processes_added) end function test_blas_config(pid, expected) - for worker in Base.Distributed.PGRP.workers + for worker in Distributed.PGRP.workers if worker.id == pid @test get(worker.config.enable_threaded_blas) == expected return @@ -1117,7 +1042,7 @@ struct ErrorSimulator <: ClusterManager mode end -function Base.launch(manager::ErrorSimulator, params::Dict, launched::Array, c::Condition) +function launch(manager::ErrorSimulator, params::Dict, launched::Array, c::Condition) exename = params[:exename] dir = params[:dir] @@ -1199,9 +1124,9 @@ global v4 = v3 # Global references to Types and Modules should work if they are locally defined global v5 = Int -global v6 = Base.Distributed +global v6 = Distributed @test remotecall_fetch(()->v5, id_other) === Int -@test remotecall_fetch(()->v6, id_other) === Base.Distributed +@test remotecall_fetch(()->v6, id_other) === Distributed struct FooStructLocal end module FooModLocal end @@ -1310,7 +1235,7 @@ wrapped_var_ser_tests() global ids_cleanup = ones(6) global ids_func = ()->ids_cleanup -clust_ser = (Base.Distributed.worker_from_id(id_other)).w_serializer +clust_ser = (Distributed.worker_from_id(id_other)).w_serializer @test remotecall_fetch(ids_func, id_other) == ids_cleanup @test haskey(clust_ser.glbs_sent, object_id(ids_cleanup)) @@ -1484,15 +1409,15 @@ struct WorkerArgTester <: ClusterManager write_cookie end -function Base.launch(manager::WorkerArgTester, params::Dict, launched::Array, c::Condition) +function launch(manager::WorkerArgTester, params::Dict, launched::Array, c::Condition) dir = params[:dir] exename = params[:exename] exeflags = params[:exeflags] - cmd = `$exename $exeflags --bind-to $(Base.Distributed.LPROC.bind_addr) $(manager.worker_opt)` + cmd = `$exename $exeflags --bind-to $(Distributed.LPROC.bind_addr) $(manager.worker_opt)` cmd = pipeline(detach(setenv(cmd, dir=dir))) io = open(cmd, "r+") - manager.write_cookie && Base.Distributed.write_cookie(io) + manager.write_cookie && Distributed.write_cookie(io) wconfig = WorkerConfig() wconfig.process = io @@ -1501,7 +1426,7 @@ function Base.launch(manager::WorkerArgTester, params::Dict, launched::Array, c: notify(c) end -Base.manage(::WorkerArgTester, ::Integer, ::WorkerConfig, ::Symbol) = nothing +manage(::WorkerArgTester, ::Integer, ::WorkerConfig, ::Symbol) = nothing nprocs()>1 && rmprocs(workers()) @@ -1509,12 +1434,12 @@ npids = addprocs_with_testenv(WorkerArgTester(`--worker`, true)) @test remotecall_fetch(myid, npids[1]) == npids[1] rmprocs(npids) -Base.cluster_cookie("") # An empty string is a valid cookie +cluster_cookie("") # An empty string is a valid cookie npids = addprocs_with_testenv(WorkerArgTester(`--worker=`, false)) @test remotecall_fetch(myid, npids[1]) == npids[1] rmprocs(npids) -Base.cluster_cookie("foobar") # custom cookie +cluster_cookie("foobar") # custom cookie npids = addprocs_with_testenv(WorkerArgTester(`--worker=foobar`, false)) @test remotecall_fetch(myid, npids[1]) == npids[1] @@ -1531,7 +1456,7 @@ function reuseport_tests() remotecall_fetch(p) do ports_lower = [] # ports of pids lower than myid() ports_higher = [] # ports of pids higher than myid() - for w in Base.Distributed.PGRP.workers + for w in Distributed.PGRP.workers w.id == myid() && continue port = Base._sockname(w.r_stream, true)[2] if (w.id == 1) @@ -1572,3 +1497,4 @@ end # cluster at any time only supports a single topology. rmprocs(workers()) include("topology.jl") + diff --git a/test/distributed.jl b/stdlib/Distributed/test/runtests.jl similarity index 84% rename from test/distributed.jl rename to stdlib/Distributed/test/runtests.jl index fc3bac479dab58..ca43d192fa49f9 100644 --- a/test/distributed.jl +++ b/stdlib/Distributed/test/runtests.jl @@ -2,8 +2,8 @@ # Run the distributed test outside of the main driver since it needs its own # set of dedicated workers. +include(joinpath(JULIA_HOME, "..", "share", "julia", "test", "testenv.jl")) -include("testenv.jl") cmd = `$test_exename $test_exeflags distributed_exec.jl` if !success(pipeline(cmd; stdout=STDOUT, stderr=STDERR)) && ccall(:jl_running_on_valgrind,Cint,()) == 0 diff --git a/test/topology.jl b/stdlib/Distributed/test/topology.jl similarity index 88% rename from test/topology.jl rename to stdlib/Distributed/test/topology.jl index 58f6ba457de011..ff898351d3f522 100644 --- a/test/topology.jl +++ b/stdlib/Distributed/test/topology.jl @@ -36,16 +36,16 @@ mutable struct TopoTestManager <: ClusterManager np::Integer end -function Base.launch(manager::TopoTestManager, params::Dict, launched::Array, c::Condition) +function launch(manager::TopoTestManager, params::Dict, launched::Array, c::Condition) dir = params[:dir] exename = params[:exename] exeflags = params[:exeflags] - cmd = `$exename $exeflags --bind-to $(Base.Distributed.LPROC.bind_addr) --worker` + cmd = `$exename $exeflags --bind-to $(Distributed.LPROC.bind_addr) --worker` cmd = pipeline(detach(setenv(cmd, dir=dir))) for i in 1:manager.np io = open(cmd, "r+") - Base.Distributed.write_cookie(io) + Distributed.write_cookie(io) wconfig = WorkerConfig() wconfig.process = io @@ -59,7 +59,7 @@ function Base.launch(manager::TopoTestManager, params::Dict, launched::Array, c: end const map_pid_ident=Dict() -function Base.manage(manager::TopoTestManager, id::Integer, config::WorkerConfig, op::Symbol) +function manage(manager::TopoTestManager, id::Integer, config::WorkerConfig, op::Symbol) if op == :register map_pid_ident[id] = get(config.ident) elseif op == :interrupt @@ -96,8 +96,8 @@ remove_workers_and_test() # test `lazy` connection setup function def_count_conn() @everywhere function count_connected_workers() - count(x -> isa(x, Base.Distributed.Worker) && isdefined(x, :r_stream) && isopen(x.r_stream), - Base.Distributed.PGRP.workers) + count(x -> isa(x, Distributed.Worker) && isdefined(x, :r_stream) && isopen(x.r_stream), + Distributed.PGRP.workers) end end diff --git a/stdlib/SharedArrays/src/SharedArrays.jl b/stdlib/SharedArrays/src/SharedArrays.jl index c5ac5ce01ae60e..17836b0e1f395b 100644 --- a/stdlib/SharedArrays/src/SharedArrays.jl +++ b/stdlib/SharedArrays/src/SharedArrays.jl @@ -5,13 +5,13 @@ Provide the [`SharedArray`](@ref) type. It represents an array, which is shared """ module SharedArrays -using Mmap, Base.Distributed +using Mmap, Distributed import Base: length, size, ndims, IndexStyle, reshape, convert, deepcopy_internal, serialize, deserialize, show, getindex, setindex!, fill!, rand!, similar, reduce, map!, copy!, unsafe_convert import Base.Random import Base.Serializer: serialize_cycle_header, serialize_type, writetag, UNDEFREF_TAG -import Base.Distributed: RRID, procs +import Distributed: RRID, procs import Base.Filesystem: JL_O_CREAT, JL_O_RDWR, S_IRUSR, S_IWUSR export SharedArray, SharedVector, SharedMatrix, sdata, indexpids, localindexes diff --git a/stdlib/SharedArrays/test/runtests.jl b/stdlib/SharedArrays/test/runtests.jl index 249e0cdc1500d3..5d9e92338ec1e6 100644 --- a/stdlib/SharedArrays/test/runtests.jl +++ b/stdlib/SharedArrays/test/runtests.jl @@ -1,12 +1,7 @@ -using Test, SharedArrays -include(joinpath(JULIA_HOME, "..", "share", "julia", "test", "testenv.jl")) +# This file is a part of Julia. License is MIT: https://julialang.org/license -# Test a few "remote" invocations when no workers are present -@test remote(myid)() == 1 -@test pmap(identity, 1:100) == [1:100...] -@test 100 == @parallel (+) for i in 1:100 - 1 - end +using Test, Distributed, SharedArrays +include(joinpath(JULIA_HOME, "..", "share", "julia", "test", "testenv.jl")) addprocs_with_testenv(4) @test nprocs() == 5 diff --git a/stdlib/Test/test/runtests.jl b/stdlib/Test/test/runtests.jl index e0e754429287c6..70381401e8f5d4 100644 --- a/stdlib/Test/test/runtests.jl +++ b/stdlib/Test/test/runtests.jl @@ -1,6 +1,6 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -using Test +using Test, Distributed @testset "@test" begin @test true diff --git a/test/abstractarray.jl b/test/abstractarray.jl index 6e852a90c5b225..df6039d4108004 100644 --- a/test/abstractarray.jl +++ b/test/abstractarray.jl @@ -633,72 +633,6 @@ Base.getindex(A::TSlowNIndexes, index::Int...) = error("Must use $(ndims(A)) ind Base.getindex(A::TSlowNIndexes{T,2}, i::Int, j::Int) where {T} = A.data[i,j] -mutable struct GenericIterator{N} end -Base.start(::GenericIterator{N}) where {N} = 1 -Base.next(::GenericIterator{N}, i) where {N} = (i, i + 1) -Base.done(::GenericIterator{N}, i) where {N} = i > N ? true : false -Base.iteratorsize(::Type{GenericIterator{N}}) where {N} = Base.SizeUnknown() - -function test_map(::Type{TestAbstractArray}) - empty_pool = WorkerPool([myid()]) - pmap_fallback = (f, c...) -> pmap(empty_pool, f, c...) - - for mapf in [map, asyncmap, pmap_fallback] - for typ in (Float16, Float32, Float64, - Int8, Int16, Int32, Int64, Int128, - UInt8, UInt16, UInt32, UInt64, UInt128), - arg_typ in (Integer, - Signed, - Unsigned) - X = typ[1:10...] - _typ = typeof(arg_typ(one(typ))) - @test mapf(arg_typ, X) == _typ[1:10...] - end - - # generic map - f(x) = x + 1 - I = GenericIterator{10}() - @test mapf(f, I) == Any[2:11...] - - # AbstractArray map for 2 arg case - f(x, y) = x + y - B = Float64[1:10...] - C = Float64[1:10...] - @test mapf(f, convert(Vector{Int},B), C) == Float64[ 2 * i for i in 1:10 ] - @test mapf(f, Int[], Float64[]) == Union{}[] - # map with different result types - let m = mapf(x->x+1, Number[1, 2.0]) - # FIXME why is this different for asyncmap? - @test mapf !== map || isa(m, Vector{Real}) - @test m == Real[2, 3.0] - end - - # AbstractArray map for N-arg case - A = Array{Int}(10) - f(x, y, z) = x + y + z - D = Float64[1:10...] - - @test map!(f, A, B, C, D) == Int[ 3 * i for i in 1:10 ] - @test mapf(f, B, C, D) == Float64[ 3 * i for i in 1:10 ] - @test mapf(f, Int[], Int[], Complex{Int}[]) == Union{}[] - end - - # In-place map - A = Float64[1:10...] - map!(x -> x*x, A, A) - @test A == map(x -> x*x, Float64[1:10...]) - B = Float64[1:10...] - Base.asyncmap!(x->x*x, B, B) - @test A == B - - # Map to destination collection - map!((x,y,z)->x*y*z, A, Float64[1:10...], Float64[1:10...], Float64[1:10...]) - @test A == map(x->x*x*x, Float64[1:10...]) - C = Base.asyncmap!((x,y,z)->x*y*z, B, Float64[1:10...], Float64[1:10...], Float64[1:10...]) - @test A == B - @test B === C -end - @testset "issue #15689, mapping an abstract type" begin @test isa(map(Set, Array[[1,2],[3,4]]), Vector{Set{Int}}) end @@ -751,7 +685,10 @@ test_setindex!_internals(TestAbstractArray) test_get(TestAbstractArray) test_cat(TestAbstractArray) test_ind2sub(TestAbstractArray) -test_map(TestAbstractArray) + +include("generic_map_tests.jl") +generic_map_tests(map, map!) + test_UInt_indexing(TestAbstractArray) test_13315(TestAbstractArray) test_checksquare() diff --git a/test/asyncmap.jl b/test/asyncmap.jl new file mode 100644 index 00000000000000..81f938451a2dea --- /dev/null +++ b/test/asyncmap.jl @@ -0,0 +1,60 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +# Test asyncmap +@test allunique(asyncmap(x->(sleep(1.0);object_id(current_task())), 1:10)) + +# num tasks +@test length(unique(asyncmap(x->(yield();object_id(current_task())), 1:20; ntasks=5))) == 5 + +# default num tasks +@test length(unique(asyncmap(x->(yield();object_id(current_task())), 1:200))) == 100 + +# ntasks as a function +let nt=0 + global nt_func + nt_func() = (v=div(nt, 25); nt+=1; v) # increment number of tasks by 1 for every 25th call. + # nt_func() will be called initally once and then for every + # iteration +end +@test length(unique(asyncmap(x->(yield();object_id(current_task())), 1:200; ntasks=nt_func))) == 7 + +# batch mode tests +let ctr=0 + global next_ctr + next_ctr() = (ctr+=1; ctr) +end +resp = asyncmap(x->(v=next_ctr(); map(_->v, x)), 1:22; ntasks=5, batch_size=5) +@test length(resp) == 22 +@test length(unique(resp)) == 5 + +input = rand(1:1000, 100) +@test asyncmap(x->map(args->identity(args...), x), input; ntasks=5, batch_size=5) == input + +# check whether shape is retained +a=rand(2,2) +b=asyncmap(identity, a) +@test a == b +@test size(a) == size(b) + +# check with an iterator that does not implement size() +c=Channel(32); foreach(i->put!(c,i), 1:10); close(c) +b=asyncmap(identity, c) +@test Int[1:10...] == b +@test size(b) == (10,) + +# check with an iterator that has only implements length() +len_only_iterable = (1,2,3,4,5) +@test Base.iteratorsize(len_only_iterable) == Base.HasLength() +@test asyncmap(identity, len_only_iterable) == map(identity, len_only_iterable) + +# Error conditions +@test_throws ArgumentError asyncmap(identity, 1:10; batch_size=0) +@test_throws ArgumentError asyncmap(identity, 1:10; batch_size="10") +@test_throws ArgumentError asyncmap(identity, 1:10; ntasks="10") + +include("generic_map_tests.jl") +generic_map_tests(asyncmap, asyncmap!) + +# asyncmap with various types. Test for equivalence with map +run_map_equivalence_tests(asyncmap) +@test asyncmap(uppercase, "Hello World!") == map(uppercase, "Hello World!") diff --git a/test/channels.jl b/test/channels.jl index 8a65ced7cd40ab..2d094d1531a32e 100644 --- a/test/channels.jl +++ b/test/channels.jl @@ -144,6 +144,7 @@ for N in [0,10] @test ref[] == nth # channeled_tasks + using Distributed for T in [Any, Int] chnls, tasks = Base.channeled_tasks(2, (c1,c2)->(assert(take!(c1)==1); put!(c2,2)); ctypes=[T,T], csizes=[N,N]) put!(chnls[1], 1) diff --git a/test/choosetests.jl b/test/choosetests.jl index cfb0a3d08a6f0a..1c40685afa2bc4 100644 --- a/test/choosetests.jl +++ b/test/choosetests.jl @@ -39,17 +39,17 @@ function choosetests(choices = []) "bigfloat", "sorting", "statistics", "spawn", "backtrace", "file", "read", "version", "resolve", "namedtuple", "mpfr", "broadcast", "complex", "socket", - "floatapprox", "stdlib", "reflection", "regex", "float16", + "floatapprox", "reflection", "regex", "float16", "combinatorics", "sysinfo", "env", "rounding", "ranges", "mod2pi", "euler", "show", "lineedit", "replcompletions", "repl", "replutil", "sets", "goto", "llvmcall", "llvmcall2", "grisu", "nullable", "meta", "stacktraces", "libgit2", "docs", "markdown", "serialize", "misc", "threads", "enums", "cmdlineargs", "i18n", "workspace", "libdl", "int", - "checked", "bitset", "floatfuncs", "compile", "distributed", "inline", + "checked", "bitset", "floatfuncs", "compile", "inline", "boundscheck", "error", "ambiguous", "cartesian", "asmvariant", "osutils", "channels", "iostream", "specificity", "codegen", "codevalidation", - "reinterpretarray", "syntax" + "reinterpretarray", "syntax", "asyncmap", "stdlib" ] if isdir(joinpath(JULIA_HOME, Base.DOCDIR, "examples")) @@ -158,7 +158,7 @@ function choosetests(choices = []) prepend!(tests, ["ambiguous"]) end - net_required_for = ["socket", "distributed", "libgit2"] + net_required_for = ["socket", "libgit2"] net_on = true try ipa = getipaddr() diff --git a/test/compile.jl b/test/compile.jl index 8a736f1cf7cd3b..72dd8df5ade7d9 100644 --- a/test/compile.jl +++ b/test/compile.jl @@ -1,6 +1,6 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -using Test +using Test, Distributed import Base: root_module @@ -218,7 +218,7 @@ try [:Base, :Core, Foo2_module, FooBase_module, :Main, :Test]), # plus modules included in the system image Dict(s => Base.module_uuid(Base.root_module(s)) for s in - [:DelimitedFiles,:Mmap,:Base64])) + [:DelimitedFiles, :Mmap, :Base64, :Distributed])) @test discard_module.(deps) == deps1 @test current_task()(0x01, 0x4000, 0x30031234) == 2 @@ -602,7 +602,7 @@ let @eval using $ModuleB uuid = Base.module_uuid(root_module(ModuleB)) for wid in test_workers - @test Base.Distributed.remotecall_eval(Main, wid, :( Base.module_uuid(Base.root_module($(QuoteNode(ModuleB)))) )) == uuid + @test Distributed.remotecall_eval(Main, wid, :( Base.module_uuid(Base.root_module($(QuoteNode(ModuleB)))) )) == uuid if wid != myid() # avoid world-age errors on the local proc @test remotecall_fetch(g, wid) == wid end diff --git a/test/generic_map_tests.jl b/test/generic_map_tests.jl new file mode 100644 index 00000000000000..52dfc9a31f50ea --- /dev/null +++ b/test/generic_map_tests.jl @@ -0,0 +1,84 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +mutable struct GenericIterator{N} end +Base.start(::GenericIterator{N}) where {N} = 1 +Base.next(::GenericIterator{N}, i) where {N} = (i, i + 1) +Base.done(::GenericIterator{N}, i) where {N} = i > N ? true : false +Base.iteratorsize(::Type{GenericIterator{N}}) where {N} = Base.SizeUnknown() + +function generic_map_tests(mapf, inplace_mapf=nothing) + for typ in (Float16, Float32, Float64, + Int8, Int16, Int32, Int64, Int128, + UInt8, UInt16, UInt32, UInt64, UInt128), + arg_typ in (Integer, + Signed, + Unsigned) + X = typ[1:10...] + _typ = typeof(arg_typ(one(typ))) + @test mapf(arg_typ, X) == _typ[1:10...] + end + + # generic map + f(x) = x + 1 + I = GenericIterator{10}() + @test mapf(f, I) == Any[2:11...] + + # AbstractArray map for 2 arg case + f(x, y) = x + y + B = Float64[1:10...] + C = Float64[1:10...] + @test mapf(f, convert(Vector{Int},B), C) == Float64[ 2 * i for i in 1:10 ] + @test mapf(f, Int[], Float64[]) == Union{}[] + # map with different result types + let m = mapf(x->x+1, Number[1, 2.0]) + @test isa(m, Vector{Real}) + @test m == Real[2, 3.0] + end + + # AbstractArray map for N-arg case + A = Array{Int}(10) + f(x, y, z) = x + y + z + D = Float64[1:10...] + + @test map!(f, A, B, C, D) == Int[ 3 * i for i in 1:10 ] + @test mapf(f, B, C, D) == Float64[ 3 * i for i in 1:10 ] + @test mapf(f, Int[], Int[], Complex{Int}[]) == Union{}[] + + # In-place map + if inplace_mapf != nothing + A = Float64[1:10...] + inplace_mapf(x -> x*x, A, A) + @test A == map(x -> x*x, Float64[1:10...]) + + # Map to destination collection + B = inplace_mapf((x,y,z)->x*y*z, A, Float64[1:10...], Float64[1:10...], Float64[1:10...]) + @test A == map(x->x*x*x, Float64[1:10...]) + @test A === B + end +end + +function testmap_equivalence(mapf, f, c...) + x1 = mapf(f,c...) + x2 = map(f,c...) + + if Base.iteratorsize == Base.HasShape() + @test size(x1) == size(x2) + else + @test length(x1) == length(x2) + end + + @test eltype(x1) == eltype(x2) + + for (v1,v2) in zip(x1,x2) + @test v1==v2 + end +end + +function run_map_equivalence_tests(mapf) + testmap_equivalence(mapf, identity, (1,2,3,4)) + testmap_equivalence(mapf, x->x>0 ? 1.0 : 0.0, sparse(sparse(1.0I, 5, 5))) + testmap_equivalence(mapf, (x,y,z)->x+y+z, 1,2,3) + testmap_equivalence(mapf, x->x ? false : true, BitArray(10,10)) + testmap_equivalence(mapf, x->"foobar", BitArray(10,10)) + testmap_equivalence(mapf, (x,y,z)->string(x,y,z), BitArray(10), ones(10), "1234567890") +end diff --git a/test/meta.jl b/test/meta.jl index f49e20c62f49ae..9fe933cb8b511b 100644 --- a/test/meta.jl +++ b/test/meta.jl @@ -130,7 +130,8 @@ show_sexpr(ioB,:(1+1)) show_sexpr(ioB,QuoteNode(1),1) -@test Base.Distributed.extract_imports(:(begin; import Foo, Bar; let; using Baz; end; end)) == +using Distributed +@test Distributed.extract_imports(:(begin; import Foo, Bar; let; using Baz; end; end)) == [:Foo, :Bar, :Baz] # test base/expr.jl diff --git a/test/runtests.jl b/test/runtests.jl index 77dd16397474b2..214bc5e6a1f226 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,6 +1,8 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license using Test +using Distributed + include("choosetests.jl") include("testenv.jl") @@ -22,9 +24,6 @@ function move_to_node1(t) end # Base.compile only works from node 1, so compile test is handled specially move_to_node1("compile") -# In a constrained memory environment, run the "distributed" test after all other tests -# since it starts a lot of workers and can easily exceed the maximum memory -max_worker_rss != typemax(Csize_t) && move_to_node1("distributed") cd(dirname(@__FILE__)) do n = 1