diff --git a/Project.toml b/Project.toml index ca1a7306cc..28b41ef476 100644 --- a/Project.toml +++ b/Project.toml @@ -8,7 +8,6 @@ version = "0.19.12" Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" Configurations = "5218b696-f38b-4ac9-8b61-a12ec717816d" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" -Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" FileWatching = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" FuzzyCompletions = "fb4132e2-a121-4a70-b8a1-d5b831dcdcc2" HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" @@ -16,12 +15,14 @@ HypertextLiteral = "ac1192a8-f4b3-4bfe-ba22-af5b92cd3ab2" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" MIMEs = "6c6e2e6c-3030-632d-7369-2d6c69616d65" +Malt = "36869731-bdee-424d-aa32-cab38c994e3b" Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" MsgPack = "99f44e22-a591-53d1-9472-aa23ef4bd671" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" PrecompileSignatures = "91cefc8d-f054-46dc-8f8c-26e11d7c5411" REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" RelocatableFolders = "05181044-ff0b-4ac5-8273-598c1e38db00" +Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" @@ -34,6 +35,7 @@ FuzzyCompletions = "0.3, 0.4, 0.5" HTTP = "^1.0.2" HypertextLiteral = "0.7, 0.8, 0.9" MIMEs = "0.1" +Malt = "^0.6.0" MsgPack = "1.1" PrecompileSignatures = "3" RelocatableFolders = "0.1, 0.2, 0.3" @@ -43,12 +45,12 @@ julia = "^1.6" [extras] DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +Memoize = "c03570c3-d221-55d1-a50c-7939bbd78826" OffsetArrays = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" TimerOutputs = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" -Memoize = "c03570c3-d221-55d1-a50c-7939bbd78826" [targets] test = ["DataFrames", "OffsetArrays", "Random", "Sockets", "Test", "TimerOutputs", "Memoize"] diff --git a/src/Configuration.jl b/src/Configuration.jl index 5e12d5b06f..31cba09114 100644 --- a/src/Configuration.jl +++ b/src/Configuration.jl @@ -128,7 +128,6 @@ Note that Pluto is quickly evolving software, maintained by designers, educators end const RUN_NOTEBOOK_ON_LOAD_DEFAULT = true -const WORKSPACE_USE_DISTRIBUTED_DEFAULT = true const LAZY_WORKSPACE_CREATION_DEFAULT = false const CAPTURE_STDOUT_DEFAULT = true @@ -138,13 +137,11 @@ const CAPTURE_STDOUT_DEFAULT = true Options to change Pluto's evaluation behaviour during internal testing. These options are not intended to be changed during normal use. - `run_notebook_on_load::Bool = $RUN_NOTEBOOK_ON_LOAD_DEFAULT` Whether to evaluate a notebook on load. -- `workspace_use_distributed::Bool = $WORKSPACE_USE_DISTRIBUTED_DEFAULT` Whether to start notebooks in a separate process. - `lazy_workspace_creation::Bool = $LAZY_WORKSPACE_CREATION_DEFAULT` - `capture_stdout::Bool = $CAPTURE_STDOUT_DEFAULT` """ @option mutable struct EvaluationOptions run_notebook_on_load::Bool = RUN_NOTEBOOK_ON_LOAD_DEFAULT - workspace_use_distributed::Bool = WORKSPACE_USE_DISTRIBUTED_DEFAULT lazy_workspace_creation::Bool = LAZY_WORKSPACE_CREATION_DEFAULT capture_stdout::Bool = CAPTURE_STDOUT_DEFAULT end @@ -250,7 +247,6 @@ function from_flat_kwargs(; require_secret_for_open_links::Bool = REQUIRE_SECRET_FOR_OPEN_LINKS_DEFAULT, require_secret_for_access::Bool = REQUIRE_SECRET_FOR_ACCESS_DEFAULT, run_notebook_on_load::Bool = RUN_NOTEBOOK_ON_LOAD_DEFAULT, - workspace_use_distributed::Bool = WORKSPACE_USE_DISTRIBUTED_DEFAULT, lazy_workspace_creation::Bool = LAZY_WORKSPACE_CREATION_DEFAULT, capture_stdout::Bool = CAPTURE_STDOUT_DEFAULT, compile::Union{Nothing,String} = COMPILE_DEFAULT, @@ -289,7 +285,6 @@ function from_flat_kwargs(; ) evaluation = EvaluationOptions(; run_notebook_on_load, - workspace_use_distributed, lazy_workspace_creation, capture_stdout, ) diff --git a/src/evaluation/WorkspaceManager.jl b/src/evaluation/WorkspaceManager.jl index 0fc39ff0e2..0bf9a01ce3 100644 --- a/src/evaluation/WorkspaceManager.jl +++ b/src/evaluation/WorkspaceManager.jl @@ -6,18 +6,18 @@ import ..Pluto.PkgCompat import ..Configuration: CompilerOptions, _merge_notebook_compiler_options, _convert_to_flags import ..Pluto.ExpressionExplorer: FunctionName import ..PlutoRunner -import Distributed +import Malt """ -Contains the Julia process (in the sense of `Distributed.addprocs`) to evaluate code in. +Contains the Julia process to evaluate code in. Each notebook gets at most one `Workspace` at any time, but it can also have no `Workspace` (it cannot `eval` code in this case). """ Base.@kwdef mutable struct Workspace - pid::Integer + worker::Malt.Worker notebook_id::UUID discarded::Bool=false - remote_log_channel::Distributed.RemoteChannel + remote_log_channel::Channel module_name::Symbol dowork_token::Token=Token() nbpkg_was_active::Bool=false @@ -36,11 +36,7 @@ const process_preamble = quote ENV["JULIA_REVISE_WORKER_ONLY"] = "1" end -const Distributed_expr = quote - Base.loaded_modules[Base.PkgId(Base.UUID("8ba89e20-285c-5b6f-9357-94700520ee1b"), "Distributed")] -end - -const workspaces = Dict{UUID,Task}() +const active_workspaces = Dict{UUID,Task}() "Set of notebook IDs that we will never make a process for again." const discarded_workspaces = Set{UUID}() @@ -49,52 +45,31 @@ const discarded_workspaces = Set{UUID}() function make_workspace((session, notebook)::SN; is_offline_renderer::Bool=false)::Workspace is_offline_renderer || (notebook.process_status = ProcessStatus.starting) - use_distributed = if is_offline_renderer - false - else - session.options.evaluation.workspace_use_distributed - end - - pid = if use_distributed - @debug "Creating workspace process" notebook.path length(notebook.cells) - create_workspaceprocess(; compiler_options=_merge_notebook_compiler_options(notebook, session.options.compiler)) - else - pid = Distributed.myid() - if !(isdefined(Main, :PlutoRunner) && Main.PlutoRunner isa Module) - # we make PlutoRunner available in Main, right now it's only defined inside this Pluto module. - @eval Main begin - PlutoRunner = $(PlutoRunner) - end - end - pid - end + @debug "Creating workspace process" notebook.path length(notebook.cells) + worker = create_workspaceprocess(;compiler_options=_merge_notebook_compiler_options(notebook, session.options.compiler)) - Distributed.remotecall_eval(Main, [pid], quote + Malt.remote_eval_wait(worker, quote PlutoRunner.notebook_id[] = $(notebook.notebook_id) end) - remote_log_channel = Core.eval(Main, quote - $(Distributed).RemoteChannel(() -> eval(quote - channel = Channel{Any}(10) - Main.PlutoRunner.setup_plutologger( - $($(notebook.notebook_id)), - channel; - make_global=$($(use_distributed)) - ) - channel - end), $pid) + remote_log_channel = Malt.worker_channel(worker, quote + channel = Channel{Any}(10) + Main.PlutoRunner.setup_plutologger( + $(notebook.notebook_id), + channel; + make_global=false # TODO(savq): Is this still necessary? + ) + channel end) - run_channel = Core.eval(Main, quote - $(Distributed).RemoteChannel(() -> eval(:(Main.PlutoRunner.run_channel)), $pid) - end) + run_channel = Malt.worker_channel(worker, :(Main.PlutoRunner.run_channel)) - module_name = create_emptyworkspacemodule(pid) + module_name = create_emptyworkspacemodule(worker) - original_LOAD_PATH, original_ACTIVE_PROJECT = Distributed.remotecall_eval(Main, pid, :(Base.LOAD_PATH, Base.ACTIVE_PROJECT[])) + original_LOAD_PATH, original_ACTIVE_PROJECT = Malt.remote_eval_fetch(worker, :(Base.LOAD_PATH, Base.ACTIVE_PROJECT[])) workspace = Workspace(; - pid, + worker, notebook_id=notebook.notebook_id, remote_log_channel, module_name, @@ -122,20 +97,16 @@ function use_nbpkg_environment((session, notebook)::SN, workspace=nothing) workspace.discarded && return workspace.nbpkg_was_active = enabled - if workspace.pid != Distributed.myid() - new_LP = enabled ? ["@", "@stdlib"] : workspace.original_LOAD_PATH - new_AP = enabled ? PkgCompat.env_dir(notebook.nbpkg_ctx) : workspace.original_ACTIVE_PROJECT + new_LP = enabled ? ["@", "@stdlib"] : workspace.original_LOAD_PATH + new_AP = enabled ? PkgCompat.env_dir(notebook.nbpkg_ctx) : workspace.original_ACTIVE_PROJECT - Distributed.remotecall_eval(Main, [workspace.pid], quote - copy!(LOAD_PATH, $(new_LP)) - Base.ACTIVE_PROJECT[] = $(new_AP) - end) - else - # TODO - end + Malt.remote_eval_wait(workspace.worker, quote + copy!(LOAD_PATH, $(new_LP)) + Base.ACTIVE_PROJECT[] = $(new_AP) + end) end -function start_relaying_self_updates((session, notebook)::SN, run_channel::Distributed.RemoteChannel) +function start_relaying_self_updates((session, notebook)::SN, run_channel::Channel) while true try next_run_uuid = take!(run_channel) @@ -151,7 +122,7 @@ function start_relaying_self_updates((session, notebook)::SN, run_channel::Distr end end -function start_relaying_logs((session, notebook)::SN, log_channel::Distributed.RemoteChannel) +function start_relaying_logs((session, notebook)::SN, log_channel::Channel) update_throttled, flush_throttled = Pluto.throttled(0.1) do Pluto.send_notebook_changes!(Pluto.ClientRequest(session=session, notebook=notebook)) end @@ -236,7 +207,7 @@ end function bump_workspace_module(session_notebook::SN) workspace = get_workspace(session_notebook) old_name = workspace.module_name - new_name = workspace.module_name = create_emptyworkspacemodule(workspace.pid) + new_name = workspace.module_name = create_emptyworkspacemodule(workspace.worker) old_name, new_name end @@ -244,13 +215,13 @@ end function possible_bond_values(session_notebook::SN, n::Symbol; get_length::Bool=false) workspace = get_workspace(session_notebook) - Distributed.remotecall_eval(Main, workspace.pid, quote + Malt.remote_eval_fetch(workspace.worker, quote PlutoRunner.possible_bond_values($(QuoteNode(n)); get_length=$(get_length)) end) end -function create_emptyworkspacemodule(pid::Integer)::Symbol - Distributed.remotecall_eval(Main, pid, quote +function create_emptyworkspacemodule(worker::Malt.Worker)::Symbol + Malt.remote_eval_fetch(worker, quote PlutoRunner.increment_current_module() end) end @@ -258,17 +229,12 @@ end # NOTE: this function only start a worker process using given # compiler options, it does not resolve paths for notebooks # compiler configurations passed to it should be resolved before this -function create_workspaceprocess(; compiler_options=CompilerOptions())::Integer - # run on proc 1 in case Pluto is being used inside a notebook process - # Workaround for "only process 1 can add/remove workers" - pid = Distributed.remotecall_eval(Main, 1, quote - $(Distributed_expr).addprocs(1; exeflags=$(_convert_to_flags(compiler_options))) |> first - end) - - Distributed.remotecall_eval(Main, [pid], process_preamble) +function create_workspaceprocess(;compiler_options=CompilerOptions())::Malt.Worker + worker = Malt.Worker(;exeflags=_convert_to_flags(compiler_options)) + Malt.remote_eval_wait(worker, process_preamble) # so that we NEVER break the workspace with an interrupt 🤕 - @async Distributed.remotecall_eval(Main, [pid], quote + Malt.remote_eval(worker, quote while true try wait() @@ -276,7 +242,7 @@ function create_workspaceprocess(; compiler_options=CompilerOptions())::Integer end end) - pid + worker end """ @@ -292,9 +258,9 @@ function get_workspace(session_notebook::SN; allow_creation::Bool=true)::Union{N end task = if !allow_creation - get(workspaces, notebook.notebook_id, nothing) + get(active_workspaces, notebook.notebook_id, nothing) else - get!(workspaces, notebook.notebook_id) do + get!(active_workspaces, notebook.notebook_id) do Task(() -> make_workspace(session_notebook)) end end @@ -313,28 +279,16 @@ function unmake_workspace(session_notebook::SN; async::Bool=false, verbose::Bool workspace.discarded = true allow_restart || push!(discarded_workspaces, notebook.notebook_id) - if workspace.pid != Distributed.myid() - filter!(p -> fetch(p.second).pid != workspace.pid, workspaces) - t = @async begin - interrupt_workspace(workspace; verbose=false) - # run on proc 1 in case Pluto is being used inside a notebook process - # Workaround for "only process 1 can add/remove workers" - Distributed.remotecall_eval(Main, 1, quote - $(Distributed_expr).rmprocs($(workspace.pid)) - end) - end - async || wait(t) - else - if !isready(workspace.dowork_token) - @error "Cannot unmake a workspace running inside the same process: the notebook is still running." - elseif verbose - @warn "Cannot unmake a workspace running inside the same process: the notebook might still be running. If you are sure that your code is not running the notebook async, then you can use the `verbose=false` keyword argument to disable this message." - end + filter!(p -> fetch(p.second).worker != workspace.worker, active_workspaces) + t = @async begin + interrupt_workspace(workspace; verbose=false) + Malt.stop(workspace.worker) end + async || wait(t) nothing end -function distributed_exception_result(ex::Base.IOError, workspace::Workspace) +function workspace_exception_result(ex::Base.IOError, workspace::Workspace) ( output_formatted=PlutoRunner.format_output(CapturedException(ex, [])), errored=true, @@ -346,12 +300,10 @@ function distributed_exception_result(ex::Base.IOError, workspace::Workspace) ) end -function distributed_exception_result(exs::CompositeException, workspace::Workspace) +function workspace_exception_result(exs::CompositeException, workspace::Workspace) ex = first(exs.exceptions) - if ex isa Distributed.RemoteException && - ex.pid == workspace.pid && - ex.captured.ex isa InterruptException + if ex.worker == workspace.worker && ex.captured.ex isa InterruptException ( output_formatted=PlutoRunner.format_output(CapturedException(InterruptException(), [])), errored=true, @@ -361,7 +313,7 @@ function distributed_exception_result(exs::CompositeException, workspace::Worksp published_objects=Dict{String,Any}(), has_pluto_hook_features=false, ) - elseif ex isa Distributed.ProcessExitedException + elseif ex isa Malt.TerminatedWorkerException ( output_formatted=PlutoRunner.format_output(CapturedException(exs, [])), errored=true, @@ -406,13 +358,8 @@ function eval_format_fetch_in_workspace( workspace = get_workspace(session_notebook) - is_on_this_process = workspace.pid == Distributed.myid() - # if multiple notebooks run on the same process, then we need to `cd` between the different notebook paths if session_notebook isa Tuple - if is_on_this_process - cd_workspace(workspace, session_notebook[2].path) - end use_nbpkg_environment(session_notebook, workspace) end @@ -421,8 +368,7 @@ function eval_format_fetch_in_workspace( # A try block (on this process) to catch an InterruptException take!(workspace.dowork_token) early_result = try - # Use [pid] instead of pid to prevent fetching output - Distributed.remotecall_eval(Main, [workspace.pid], quote + Malt.remote_eval_wait(workspace.worker, quote PlutoRunner.run_expression( getfield(Main, $(QuoteNode(workspace.module_name))), $(QuoteNode(expr)), @@ -431,7 +377,7 @@ function eval_format_fetch_in_workspace( $function_wrapped_info, $forced_expr_id; user_requested_run=$user_requested_run, - capture_stdout=$(capture_stdout && !is_on_this_process), + capture_stdout=$capture_stdout, ) end) put!(workspace.dowork_token) @@ -439,7 +385,7 @@ function eval_format_fetch_in_workspace( catch e # Don't use a `finally` because the token needs to be back asap for the interrupting code to pick it up. put!(workspace.dowork_token) - distributed_exception_result(e, workspace) + workspace_exception_result(e, workspace) end if early_result === nothing @@ -453,7 +399,7 @@ end function eval_in_workspace(session_notebook::Union{SN,Workspace}, expr) workspace = get_workspace(session_notebook) - Distributed.remotecall_eval(Main, [workspace.pid], quote + Malt.remote_eval_wait(workspace.worker, quote Core.eval($(workspace.module_name), $(QuoteNode(expr))) end) nothing @@ -473,7 +419,7 @@ function format_fetch_in_workspace( # we format the cell output on the worker, and fetch the formatted output. withtoken(workspace.dowork_token) do try - Distributed.remotecall_eval(Main, workspace.pid, quote + Malt.remote_eval_fetch(workspace.worker, quote PlutoRunner.formatted_result_of( $(workspace.notebook_id), $cell_id, @@ -484,7 +430,7 @@ function format_fetch_in_workspace( ) end) catch e - distributed_exception_result(CompositeException([e]), workspace) + workspace_exception_result(CompositeException([e]), workspace) end end end @@ -492,7 +438,7 @@ end function collect_soft_definitions(session_notebook::SN, modules::Set{Expr}) workspace = get_workspace(session_notebook) - Distributed.remotecall_eval(Main, workspace.pid, quote + Malt.remote_eval_fetch(workspace.worker, quote PlutoRunner.collect_soft_definitions($(workspace.module_name), $modules) end) end @@ -501,7 +447,7 @@ function macroexpand_in_workspace(session_notebook::Union{SN,Workspace}, macroca workspace = get_workspace(session_notebook) module_name = module_name === nothing ? workspace.module_name : module_name - Distributed.remotecall_eval(Main, workspace.pid, quote + Malt.remote_eval_fetch(workspace.worker, quote try (true, PlutoRunner.try_macroexpand($module_name, $(workspace.notebook_id), $cell_id, $(QuoteNode(macrocall)))) catch error @@ -521,7 +467,7 @@ end function eval_fetch_in_workspace(session_notebook::Union{SN,Workspace}, expr) workspace = get_workspace(session_notebook) - Distributed.remotecall_eval(Main, workspace.pid, quote + Malt.remote_eval_fetch(workspace.worker, quote Core.eval($(workspace.module_name), $(QuoteNode(expr))) end) end @@ -529,7 +475,7 @@ end function do_reimports(session_notebook::Union{SN,Workspace}, module_imports_to_move::Set{Expr}) workspace = get_workspace(session_notebook) - Distributed.remotecall_eval(Main, [workspace.pid], quote + Malt.remote_eval_wait(workspace.worker, quote PlutoRunner.do_reimports($(workspace.module_name), $module_imports_to_move) end) end @@ -551,7 +497,7 @@ function move_vars( workspace = get_workspace(session_notebook) new_workspace_name = something(new_workspace_name, workspace.module_name) - Distributed.remotecall_eval(Main, [workspace.pid], quote + Malt.remote_eval_wait(workspace.worker, quote PlutoRunner.move_vars( $(QuoteNode(old_workspace_name)), $(QuoteNode(new_workspace_name)), @@ -638,11 +584,6 @@ function interrupt_workspace(session_notebook::Union{SN,Workspace}; verbose=true return false end - if workspace.pid == Distributed.myid() - verbose && @warn """Cells in this workspace can't be stopped, because it is not running in a separate workspace. Use `ENV["PLUTO_WORKSPACE_USE_DISTRIBUTED"]` to control whether future workspaces are generated in a separate process.""" - return false - end - if isready(workspace.dowork_token) verbose && @info "Tried to stop idle workspace - ignoring." return true @@ -653,8 +594,8 @@ function interrupt_workspace(session_notebook::Union{SN,Workspace}; verbose=true # TODO: this will also kill "pending" evaluations, and any evaluations started within 100ms of the kill. A global "evaluation count" would fix this. # TODO: listen for the final words of the remote process on stdout/stderr: "Force throwing a SIGINT" try - verbose && @info "Sending interrupt to process $(workspace.pid)" - Distributed.interrupt(workspace.pid) + verbose && @info "Sending interrupt to process $(workspace.worker)" + Malt.interrupt(workspace.worker) if poll(() -> isready(workspace.dowork_token), 5.0, 5/100) verbose && println("Cell interrupted!") @@ -665,7 +606,7 @@ function interrupt_workspace(session_notebook::Union{SN,Workspace}; verbose=true while !isready(workspace.dowork_token) for _ in 1:5 verbose && print(" 🔥 ") - Distributed.interrupt(workspace.pid) + Malt.interrupt(workspace.worker) sleep(0.18) if isready(workspace.dowork_token) break diff --git a/src/runner/PlutoRunner.jl b/src/runner/PlutoRunner.jl index 50da8c8507..d825fcc663 100644 --- a/src/runner/PlutoRunner.jl +++ b/src/runner/PlutoRunner.jl @@ -1,17 +1,21 @@ # Will be evaluated _inside_ the workspace process. -# Pluto does most things on process 1 (the server), and it uses little workspace processes to evaluate notebook code in. -# These baby processes don't import Pluto, they only import this module. Functions from this module are called by WorkspaceManager.jl via Distributed +# Pluto does most things on the server, but it uses worker processes to evaluate notebook code in. +# These processes don't import Pluto, they only import this module. +# Functions from this module are called by WorkspaceManager.jl via Malt. -# So when reading this file, pretend that you are living in process 2, and you are communicating with Pluto's server, who lives in process 1. +# When reading this file, pretend that you are living in a worker process, +# and you are communicating with Pluto's server, who lives in the main process. # The package environment that this file is loaded with is the NotebookProcessProject.toml file in this directory. # SOME EXTRA NOTES # 1. The entire PlutoRunner should be a single file. -# 2. We restrict the communication between this PlutoRunner and the Pluto server to only use *Base Julia types*, like `String`, `Dict`, `NamedTuple`, etc. +# 2. Restrict the communication between this PlutoRunner and the Pluto server to only use *Base Julia types*, like `String`, `Dict`, `NamedTuple`, etc. -# These restriction are there to allow flexibility in the way that this file is loaded on a runner process, which is something that we might want to change in the future, like when we make the transition to our own Distributed. +# These restriction are there to allow flexibility in the way that this file is +# loaded on a runner process, which is something that we might want to change +# in the future. module PlutoRunner @@ -21,7 +25,6 @@ import InteractiveUtils using Markdown import Markdown: html, htmlinline, LaTeX, withtag, htmlesc -import Distributed import Base64 import FuzzyCompletions: Completion, BslashCompletion, ModuleCompletion, PropertyCompletion, FieldCompletion, PathCompletion, DictCompletion, completions, completion_text, score import Base: show, istextmime @@ -32,7 +35,7 @@ import REPL export @bind -# This is not a struct to make it easier to pass these objects between distributed processes. +# This is not a struct to make it easier to pass these objects between processes. const MimedOutput = Tuple{Union{String,Vector{UInt8},Dict{Symbol,Any}},MIME} const ObjectID = typeof(objectid("hello computer")) @@ -846,7 +849,7 @@ const table_column_display_limit_increase = 30 const tree_display_extra_items = Dict{UUID,Dict{ObjectDimPair,Int64}}() -# This is not a struct to make it easier to pass these objects between distributed processes. +# This is not a struct to make it easier to pass these objects between processes. const FormattedCellResult = NamedTuple{(:output_formatted, :errored, :interrupted, :process_exited, :runtime, :published_objects, :has_pluto_hook_features),Tuple{PlutoRunner.MimedOutput,Bool,Bool,Bool,Union{UInt64,Nothing},Dict{String,Any},Bool}} function formatted_result_of( @@ -1872,15 +1875,15 @@ function possible_bond_values(s::Symbol; get_length::Bool=false) try length(possible_values) catch - length(make_distributed_serializable(possible_values)) + length(make_serializable(possible_values)) end : - make_distributed_serializable(possible_values) + make_serializable(possible_values) end end -make_distributed_serializable(x::Any) = x -make_distributed_serializable(x::Union{AbstractVector,AbstractSet,Base.Generator}) = collect(x) -make_distributed_serializable(x::Union{Vector,Set,OrdinalRange}) = x +make_serializable(x::Any) = x +make_serializable(x::Union{AbstractVector,AbstractSet,Base.Generator}) = collect(x) +make_serializable(x::Union{Vector,Set,OrdinalRange}) = x """ diff --git a/src/webserver/REPLTools.jl b/src/webserver/REPLTools.jl index d6a09b7c93..857b8752e1 100644 --- a/src/webserver/REPLTools.jl +++ b/src/webserver/REPLTools.jl @@ -1,5 +1,5 @@ import FuzzyCompletions: complete_path, completion_text, score -import Distributed +import Malt import .PkgCompat: package_completions using Markdown import REPL @@ -83,10 +83,13 @@ responses[:complete] = function response_complete(🙋::ClientRequest) if will_run_code(🙋.notebook) && workspace isa WorkspaceManager.Workspace && isready(workspace.dowork_token) # we don't use eval_format_fetch_in_workspace because we don't want the output to be string-formatted. # This works in this particular case, because the return object, a `Completion`, exists in this scope too. - Distributed.remotecall_eval(Main, workspace.pid, :(PlutoRunner.completion_fetcher( - $query, $pos, - getfield(Main, $(QuoteNode(workspace.module_name))), - ))) + Malt.remote_eval_fetch(workspace.worker, quote + PlutoRunner.completion_fetcher( + $query, + $pos, + getfield(Main, $(QuoteNode(workspace.module_name))), + ) + end) else # We can at least autocomplete general julia things: PlutoRunner.completion_fetcher(query, pos, Main) @@ -125,10 +128,12 @@ responses[:docs] = function response_docs(🙋::ClientRequest) workspace = WorkspaceManager.get_workspace((🙋.session, 🙋.notebook); allow_creation=false) if will_run_code(🙋.notebook) && workspace isa WorkspaceManager.Workspace && isready(workspace.dowork_token) - Distributed.remotecall_eval(Main, workspace.pid, :(PlutoRunner.doc_fetcher( - $query, - getfield(Main, $(QuoteNode(workspace.module_name))), - ))) + Malt.remote_eval_fetch(workspace.worker, quote + PlutoRunner.doc_fetcher( + $query, + getfield(Main, $(QuoteNode(workspace.module_name))), + ) + end) else (nothing, :⌛) end diff --git a/test/Bonds.jl b/test/Bonds.jl index d8469e2b16..e677489adf 100644 --- a/test/Bonds.jl +++ b/test/Bonds.jl @@ -1,17 +1,15 @@ using Test import Pluto import Pluto: update_run!, WorkspaceManager, ClientSession, ServerSession, Notebook, Cell -import Distributed +import Malt @testset "Bonds" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false fakeclient = ClientSession(:fake, nothing) 🍭.connected_clients[fakeclient.id] = fakeclient @testset "AbstractPlutoDingetjes.jl" begin - 🍭.options.evaluation.workspace_use_distributed = true notebook = Notebook([ # 1 Cell(""" @@ -248,7 +246,7 @@ import Distributed @test Pluto.possible_bond_values(🍭, notebook, :x_new) == [1,2,3] - @test_throws Exception Pluto.possible_bond_values(🍭, notebook, :asdfasdfx_new) + @test Pluto.possible_bond_values(🍭, notebook, :asdfasdfx_new) == KeyError(:asdfasdfx_new) @test Pluto.possible_bond_values(🍭, notebook, :pv1) == :NotGiven @test Pluto.possible_bond_values(🍭, notebook, :pv2) == :InfinitePossibilities @test Pluto.possible_bond_values(🍭, notebook, :pv3) == [1,2,3] @@ -302,14 +300,13 @@ import Distributed WorkspaceManager.unmake_workspace((🍭, notebook)) - 🍭.options.evaluation.workspace_use_distributed = false # test that the notebook file is runnable: - test_proc = Distributed.addprocs(1)[1] + test_worker = Malt.Worker() - Distributed.remotecall_eval(Main, test_proc, quote + Malt.remote_eval_fetch(test_worker, quote import Pkg try Pkg.UPDATED_REGISTRY_THIS_SESSION[] = true @@ -317,19 +314,17 @@ import Distributed Pkg.activate(mktempdir()) Pkg.add("AbstractPlutoDingetjes") end) - @test Distributed.remotecall_eval(Main, test_proc, quote + @test Malt.remote_eval_fetch(test_worker, quote include($(notebook.path)) true end) - Distributed.rmprocs(test_proc) + Malt.stop(test_worker) end @testset "Dependent Bound Variables" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false fakeclient = ClientSession(:fake, nothing) 🍭.connected_clients[fakeclient.id] = fakeclient - 🍭.options.evaluation.workspace_use_distributed = true notebook = Notebook([ Cell(raw"""@bind x HTML("")"""), Cell(raw"""@bind y HTML("")"""), diff --git a/test/Configuration.jl b/test/Configuration.jl index c45d0751d0..4cc5bfc662 100644 --- a/test/Configuration.jl +++ b/test/Configuration.jl @@ -59,7 +59,7 @@ end @testset "Authentication" begin port = 1238 - options = Pluto.Configuration.from_flat_kwargs(; port, launch_browser=false, workspace_use_distributed=false) + options = Pluto.Configuration.from_flat_kwargs(; port, launch_browser=false) 🍭 = Pluto.ServerSession(; options) fakeclient = ClientSession(:fake, nothing) 🍭.connected_clients[fakeclient.id] = fakeclient @@ -142,21 +142,21 @@ end server_running() = HTTP.get(local_url("favicon.ico")).status == 200 && HTTP.get(local_url("edit")).status == 200 # without notebook at startup - server_task = @async Pluto.run(port=port, launch_browser=false, workspace_use_distributed=false, require_secret_for_access=false, require_secret_for_open_links=false) + server_task = @async Pluto.run(port=port, launch_browser=false, require_secret_for_access=false, require_secret_for_open_links=false) @test poll(5) do server_running() end @async schedule(server_task, InterruptException(); error=true) # with a single notebook at startup - server_task = @async Pluto.run(notebook=first(nbnames), port=port, launch_browser=false, workspace_use_distributed=false, require_secret_for_access=false, require_secret_for_open_links=false) + server_task = @async Pluto.run(notebook=first(nbnames), port=port, launch_browser=false, require_secret_for_access=false, require_secret_for_open_links=false) @test poll(5) do server_running() end @async schedule(server_task, InterruptException(); error=true) # with multiple notebooks at startup - server_task = @async Pluto.run(notebook=nbnames, port=port, launch_browser=false, workspace_use_distributed=false, require_secret_for_access=false, require_secret_for_open_links=false) + server_task = @async Pluto.run(notebook=nbnames, port=port, launch_browser=false, require_secret_for_access=false, require_secret_for_open_links=false) @test poll(5) do server_running() end diff --git a/test/DependencyCache.jl b/test/DependencyCache.jl index 96d69bbd6d..33434bba86 100644 --- a/test/DependencyCache.jl +++ b/test/DependencyCache.jl @@ -5,8 +5,6 @@ using Pluto: update_run!, ServerSession, ClientSession, Cell, Notebook @testset "CellDepencencyVisualization" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false - fakeclient = ClientSession(:fake, nothing) 🍭.connected_clients[fakeclient.id] = fakeclient diff --git a/test/Dynamic.jl b/test/Dynamic.jl index b3bc9d4b44..e3aa7679aa 100644 --- a/test/Dynamic.jl +++ b/test/Dynamic.jl @@ -181,7 +181,6 @@ end @testset "PlutoRunner API" begin fakeclient = ClientSession(:fake, nothing) 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = true 🍭.connected_clients[fakeclient.id] = fakeclient notebook = Notebook([ diff --git a/test/Events.jl b/test/Events.jl index 990b223b6e..2ec96738b5 100644 --- a/test/Events.jl +++ b/test/Events.jl @@ -14,7 +14,6 @@ import UUIDs: UUID end 🍭 = ServerSession() 🍭.options.server.on_event = test_listener - 🍭.options.evaluation.workspace_use_distributed = false fakeclient = ClientSession(:fake, nothing) 🍭.connected_clients[fakeclient.id] = fakeclient @@ -39,4 +38,4 @@ import UUIDs: UUID # Pluto.save_notebook(io::IOBuffer, notebook): saves notebook to IO # Pluto.ServerSession(;options, event_listener) -end \ No newline at end of file +end diff --git a/test/Logging.jl b/test/Logging.jl index 1a29535d8f..8a4c2e3e97 100644 --- a/test/Logging.jl +++ b/test/Logging.jl @@ -5,7 +5,6 @@ using Pluto.WorkspaceManager: poll @testset "Logging" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = true fakeclient = ClientSession(:fake, nothing) 🍭.connected_clients[fakeclient.id] = fakeclient diff --git a/test/MacroAnalysis.jl b/test/MacroAnalysis.jl index d96d15cd82..b62a54aeac 100644 --- a/test/MacroAnalysis.jl +++ b/test/MacroAnalysis.jl @@ -5,7 +5,6 @@ import Memoize: @memoize @testset "Macro analysis" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false fakeclient = ClientSession(:fake, nothing) 🍭.connected_clients[fakeclient.id] = fakeclient @@ -30,6 +29,8 @@ import Memoize: @memoize @test cell(3) |> noerror @test :Fruit ∈ notebook.topology.nodes[cell(3)].references + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "User defined macro 1" begin @@ -51,6 +52,8 @@ import Memoize: @memoize # Works on second time because of old workspace @test :x ∈ notebook.topology.nodes[cell(2)].definitions @test Symbol("@my_assign") ∈ notebook.topology.nodes[cell(2)].references + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "User defined macro 2" begin @@ -80,6 +83,8 @@ import Memoize: @memoize @test cell(1) |> noerror @test cell(2) |> noerror @test cell(3) |> noerror + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "User defined macro 3" begin @@ -103,6 +108,8 @@ import Memoize: @memoize update_run!(🍭, notebook, cell(1)) @test cell(2) |> noerror + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "User defined macro 4" begin @@ -117,6 +124,8 @@ import Memoize: @memoize update_run!(🍭, notebook, notebook.cells) @test Symbol("@my_assign") ∈ notebook.topology.nodes[cell(2)].references + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "User defined macro 5" begin @@ -133,6 +142,8 @@ import Memoize: @memoize @test :a ∉ references(2) @test :b ∉ references(2) @test :c ∉ references(2) + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "User defined macro 6" begin @@ -153,6 +164,8 @@ import Memoize: @memoize @test [Symbol("@my_macro"), :x, :y] ⊆ notebook.topology.nodes[cell(2)].references @test cell(3).output.body == "3" + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Function docs" begin @@ -169,6 +182,8 @@ import Memoize: @memoize @test :f ∈ notebook.topology.nodes[cell(1)].funcdefs_without_signatures @test :f ∈ notebook.topology.nodes[cell(2)].references + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Expr sanitization" begin @@ -214,6 +229,8 @@ import Memoize: @memoize @test cell(2).output.body == "true" @test all(noerror, notebook.cells) + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Reverse order" begin @@ -241,6 +258,8 @@ import Memoize: @memoize @test cell(2) |> noerror @test cell(3) |> noerror @test cell(1).output.body == "\"yay\"" + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "@a defines @b" begin @@ -275,6 +294,8 @@ import Memoize: @memoize @test cell(3) |> noerror @test cell(4) |> noerror @test cell(1).output.body == "42" + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Removing macros undefvar errors dependent cells" begin @@ -296,6 +317,8 @@ import Memoize: @memoize @test notebook.cells[end].errored @test occursinerror("UndefVarError: @m", notebook.cells[end]) + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Redefines macro with new SymbolsState" begin @@ -345,6 +368,8 @@ import Memoize: @memoize # See Run.jl#resolve_topology. @test cell(4).output.body == "42" @test cell(3).errored == true + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Reactive macro update does not invalidate the macro calls" begin @@ -381,6 +406,8 @@ import Memoize: @memoize @test cell(4).output.body != "42" @test cell(4).errored == true @test cell(5) |> noerror + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Explicitely running macrocalls updates the reactive node" begin @@ -412,6 +439,8 @@ import Memoize: @memoize @test cell(4).errored == true @test cell(5) |> noerror + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Implicitely running macrocalls updates the reactive node" begin @@ -450,6 +479,8 @@ import Memoize: @memoize # an explicit run of @b() must be done. @test cell(4).output.body == output_1 @test cell(5).errored == true + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Weird behavior" begin @@ -477,6 +508,8 @@ import Memoize: @memoize @test cell(3) |> noerror @test cell(3).output.body == "1234" + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @@ -493,6 +526,8 @@ import Memoize: @memoize # x ("@b(x)") was run. Should it? Maybe set a higher precedence to cells that define # macros inside the notebook. @test_broken noerror(notebook.cells[1]; verbose=false) + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "@a defines @b initial loading" begin @@ -516,6 +551,8 @@ import Memoize: @memoize @test cell(3) |> noerror @test cell(4) |> noerror @test cell(1).output.body == "42" + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Macro with long compile time gets function wrapped" begin @@ -562,11 +599,11 @@ import Memoize: @memoize @test cell(1) |> noerror @test output_3 != cell(1).output.body + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Macro Prefix" begin - 🍭.options.evaluation.workspace_use_distributed = true - notebook = Notebook(Cell.([ "@sprintf \"answer = %d\" x", "x = y+1", @@ -595,9 +632,9 @@ import Memoize: @memoize @test cell(1) |> noerror WorkspaceManager.unmake_workspace((🍭, notebook)) - 🍭.options.evaluation.workspace_use_distributed = false end + @testset "Package macro 1" begin notebook = Notebook([ Cell("using Dates"), @@ -623,11 +660,11 @@ import Memoize: @memoize @test cell(1) |> noerror @test cell(2) |> noerror + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Package macro 2" begin - 🍭.options.evaluation.workspace_use_distributed = true - notebook = Notebook([ Cell("z = x^2 + y"), Cell("@variables x y"), @@ -677,8 +714,6 @@ import Memoize: @memoize @test cell(2) |> noerror WorkspaceManager.unmake_workspace((🍭, notebook)) - - 🍭.options.evaluation.workspace_use_distributed = false end @testset "Previous workspace for unknowns" begin @@ -702,6 +737,8 @@ import Memoize: @memoize module_from_cell3 = cell(3).output.body @test module_from_cell2 == module_from_cell3 + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Definitions" begin @@ -728,6 +765,8 @@ import Memoize: @memoize @test ":world" == cell(3).output.body @test ":world" == cell(4).output.body + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Is just text macros" begin @@ -742,6 +781,8 @@ import Memoize: @memoize update_run!(🍭, notebook, notebook.cells) @test isempty(notebook.topology.unresolved_cells) + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Macros using import" begin @@ -760,6 +801,8 @@ import Memoize: @memoize @test :option_type ∈ notebook.topology.nodes[cell(1)].references @test cell(1) |> noerror + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "GlobalRefs in macros should be respected" begin @@ -784,6 +827,8 @@ import Memoize: @memoize @test all(cell.([1,2,3]) .|> noerror) @test cell(3).output.body == "20" + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "GlobalRefs shouldn't break unreached undefined references" begin @@ -809,6 +854,8 @@ import Memoize: @memoize @test all(cell.([1,2]) .|> noerror) @test cell(2).output.body == ":this_should_be_returned" + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Doc strings" begin @@ -881,11 +928,12 @@ import Memoize: @memoize update_run!(🍭, notebook, bool) @test !occursin("An empty conjugate", bool.output.body) @test occursin("complex conjugate", bool.output.body) + + WorkspaceManager.unmake_workspace((🍭, notebook)) end @testset "Delete methods from macros" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false fakeclient = ClientSession(:fake, nothing) 🍭.connected_clients[fakeclient.id] = fakeclient @@ -928,5 +976,7 @@ import Memoize: @memoize @test occursinerror("UndefVarError: custom_func", cell(4)) @test :memoized_func ∉ notebook.topology.nodes[cell(5)].funcdefs_without_signatures @test occursinerror("UndefVarError: memoized_func", cell(6)) + + WorkspaceManager.unmake_workspace((🍭, notebook)) end end diff --git a/test/Notebook.jl b/test/Notebook.jl index 24ee129a0c..3072dbcc28 100644 --- a/test/Notebook.jl +++ b/test/Notebook.jl @@ -208,7 +208,6 @@ end @testset "Cell Metadata" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false fakeclient = ClientSession(:fake, nothing) 🍭.connected_clients[fakeclient.id] = fakeclient @@ -245,7 +244,6 @@ end @testset "Notebook Metadata" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false fakeclient = ClientSession(:fake, nothing) 🍭.connected_clients[fakeclient.id] = fakeclient @@ -272,7 +270,6 @@ end @testset "Skip as script" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false fakeclient = ClientSession(:fake, nothing) 🍭.connected_clients[fakeclient.id] = fakeclient diff --git a/test/React.jl b/test/React.jl index a4b112a02c..865395e38d 100644 --- a/test/React.jl +++ b/test/React.jl @@ -1,18 +1,15 @@ using Test import Pluto: Configuration, Notebook, ServerSession, ClientSession, update_run!, Cell, WorkspaceManager import Pluto.Configuration: Options, EvaluationOptions -import Distributed +import Malt @testset "Reactivity" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false fakeclient = ClientSession(:fake, nothing) 🍭.connected_clients[fakeclient.id] = fakeclient - @testset "Basic $(parallel ? "distributed" : "single-process")" for parallel in [false, true] - 🍭.options.evaluation.workspace_use_distributed = parallel - + @testset "Basic" begin notebook = Notebook([ Cell("x = 1"), Cell("y = x"), @@ -30,7 +27,7 @@ import Distributed ]) fakeclient.connected_notebook = notebook - @test !haskey(WorkspaceManager.workspaces, notebook.notebook_id) + @test !haskey(WorkspaceManager.active_workspaces, notebook.notebook_id) update_run!(🍭, notebook, notebook.cells[1:2]) @test notebook.cells[1].output.body == notebook.cells[2].output.body @@ -74,18 +71,12 @@ import Distributed @test notebook.cells[6].output.body == "3" update_run!(🍭, notebook, notebook.cells[7:8]) - @test if parallel - notebook.cells[8].output.body != string(Distributed.myid()) - else - notebook.cells[8].output.body == string(Distributed.myid()) - end + notebook.cells[8].output.body == "1" WorkspaceManager.unmake_workspace((🍭, notebook); verbose=false) end - 🍭.options.evaluation.workspace_use_distributed = false - @testset "Mutliple assignments" begin notebook = Notebook([ Cell("x = 1"), @@ -212,8 +203,6 @@ import Distributed # PlutoTest.jl is only working on Julia version >= 1.6 @testset "Test Firebasey" begin - 🍭.options.evaluation.workspace_use_distributed = true - file = tempname() write(file, read(normpath(Pluto.project_relative_path("src", "webserver", "Firebasey.jl")))) @@ -228,7 +217,6 @@ import Distributed @test all(noerror, notebook.cells) WorkspaceManager.unmake_workspace((🍭, notebook)) - 🍭.options.evaluation.workspace_use_distributed = false end @testset "Pkg topology workarounds" begin @@ -380,8 +368,6 @@ import Distributed end @testset "Reactive usings 4" begin - 🍭.options.evaluation.workspace_use_distributed = true - notebook = Notebook([ Cell("@sprintf \"double_december = %d\" double_december"), Cell("double_december = 2December"), @@ -407,7 +393,6 @@ import Distributed @test notebook.cells[1].output.body == "\"double_december = 24\"" WorkspaceManager.unmake_workspace((🍭, notebook)) - 🍭.options.evaluation.workspace_use_distributed = false end @testset "Reactive usings 5" begin @@ -440,8 +425,6 @@ import Distributed end @testset "Function dependencies" begin - 🍭.options.evaluation.workspace_use_distributed = true - notebook = Notebook(Cell.([ "a'b", "import LinearAlgebra", @@ -458,7 +441,6 @@ import Distributed @test notebook.cells[1].output.body == "200" WorkspaceManager.unmake_workspace((🍭, notebook)) - 🍭.options.evaluation.workspace_use_distributed = false end @testset "Function use inv in its def but also has a method on inv" begin diff --git a/test/ReloadFromFile.jl b/test/ReloadFromFile.jl index 06277a77cd..05e0c090fd 100644 --- a/test/ReloadFromFile.jl +++ b/test/ReloadFromFile.jl @@ -1,7 +1,6 @@ using Test import Pluto: Configuration, Notebook, ServerSession, ClientSession, update_run!, Cell, WorkspaceManager, SessionActions, save_notebook import Pluto.Configuration: Options, EvaluationOptions -import Distributed using Pluto.WorkspaceManager: poll import Pkg @@ -23,7 +22,6 @@ end retry(3) do 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false 🍭.options.server.auto_reload_from_file = true @@ -165,4 +163,4 @@ end @assert notebook.nbpkg_restart_required_msg !== nothing end @test true -end \ No newline at end of file +end diff --git a/test/RichOutput.jl b/test/RichOutput.jl index 26d1abb445..7bbb66abb7 100644 --- a/test/RichOutput.jl +++ b/test/RichOutput.jl @@ -6,7 +6,6 @@ import Pluto: update_run!, WorkspaceManager, ClientSession, ServerSession, Noteb @testset "Rich output" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false fakeclient = ClientSession(:fake, nothing) 🍭.connected_clients[fakeclient.id] = fakeclient @@ -138,8 +137,6 @@ import Pluto: update_run!, WorkspaceManager, ClientSession, ServerSession, Noteb @testset "Special arrays" begin - 🍭.options.evaluation.workspace_use_distributed = true - notebook = Notebook([ Cell("using OffsetArrays"), Cell("OffsetArray(zeros(3), 20:22)"), @@ -195,7 +192,6 @@ import Pluto: update_run!, WorkspaceManager, ClientSession, ServerSession, Noteb @test occursin("103", s) WorkspaceManager.unmake_workspace((🍭, notebook)) - 🍭.options.evaluation.workspace_use_distributed = false end @testset "Circular references" begin @@ -236,7 +232,6 @@ import Pluto: update_run!, WorkspaceManager, ClientSession, ServerSession, Noteb end @testset "Table viewer" begin - 🍭.options.evaluation.workspace_use_distributed = true notebook = Notebook([ Cell("using DataFrames, Tables"), Cell("DataFrame()"), @@ -315,7 +310,6 @@ import Pluto: update_run!, WorkspaceManager, ClientSession, ServerSession, Noteb # TODO: test lazy loading more rows/cols WorkspaceManager.unmake_workspace((🍭, notebook)) - 🍭.options.evaluation.workspace_use_distributed = false end begin diff --git a/test/WorkspaceManager.jl b/test/WorkspaceManager.jl index 218587e1e6..ad29ae8ed8 100644 --- a/test/WorkspaceManager.jl +++ b/test/WorkspaceManager.jl @@ -2,7 +2,7 @@ using Test using Pluto.Configuration: CompilerOptions using Pluto.WorkspaceManager: _merge_notebook_compiler_options import Pluto: update_save_run!, update_run!, WorkspaceManager, ClientSession, ServerSession, Notebook, Cell, project_relative_path -import Distributed +import Malt @testset "Workspace manager" begin # basic functionality is already tested by the reactivity tests @@ -12,7 +12,6 @@ import Distributed fakeclientA = ClientSession(:fakeA, nothing) fakeclientB = ClientSession(:fakeB, nothing) 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = true 🍭.connected_clients[fakeclientA.id] = fakeclientA 🍭.connected_clients[fakeclientB.id] = fakeclientB @@ -44,7 +43,6 @@ import Distributed @testset "Variables with secret names" begin fakeclient = ClientSession(:fake, nothing) 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false 🍭.connected_clients[fakeclient.id] = fakeclient notebook = Notebook([ @@ -68,7 +66,7 @@ import Distributed client = ClientSession(:fakeA, nothing) 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = true + 🍭.options.evaluation.capture_stdout = false 🍭.connected_clients[client.id] = client notebook = Notebook([ @@ -102,17 +100,10 @@ import Distributed update_run!(🍭, notebook, notebook.cells[5]) @test notebook.cells[5] |> noerror - - desired_nprocs = Distributed.nprocs() - 1 setcode!(notebook.cells[5], "Pluto.SessionActions.shutdown(s, nb)") update_run!(🍭, notebook, notebook.cells[5]) @test noerror(notebook.cells[5]) - while Distributed.nprocs() != desired_nprocs - sleep(.1) - end - sleep(.1) - WorkspaceManager.unmake_workspace((🍭, notebook)) end end diff --git a/test/cell_disabling.jl b/test/cell_disabling.jl index 19bc64aa83..65e7bfe7f1 100644 --- a/test/cell_disabling.jl +++ b/test/cell_disabling.jl @@ -4,7 +4,6 @@ using Pluto: update_run!, ServerSession, ClientSession, Cell, Notebook @testset "Cell Disabling" begin 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false fakeclient = ClientSession(:fake, nothing) 🍭.connected_clients[fakeclient.id] = fakeclient diff --git a/test/compiletimes.jl b/test/compiletimes.jl index c1eef7d165..9f43543aad 100644 --- a/test/compiletimes.jl +++ b/test/compiletimes.jl @@ -30,7 +30,6 @@ end 🍭 = Pluto.ServerSession() 🍭.options.server.disable_writing_notebook_files = true -🍭.options.evaluation.workspace_use_distributed = false path = joinpath(pkgdir(Pluto), "sample", "Basic.jl") @@ -45,7 +44,7 @@ HTTP.get("http://github.com") @timeit TOUT "Pluto.run" server_task = @eval let port = 13435 - options = Pluto.Configuration.from_flat_kwargs(; port, launch_browser=false, workspace_use_distributed=false, require_secret_for_access=false, require_secret_for_open_links=false) + options = Pluto.Configuration.from_flat_kwargs(; port, launch_browser=false, require_secret_for_access=false, require_secret_for_open_links=false) 🍭 = Pluto.ServerSession(; options) server_task = @async Pluto.run(🍭) diff --git a/test/helpers.jl b/test/helpers.jl index ee5c51b3e5..f574f9ebc8 100644 --- a/test/helpers.jl +++ b/test/helpers.jl @@ -16,7 +16,6 @@ import Pluto.ExpressionExplorer: SymbolsState, compute_symbolreferences, Functio using Sockets using Test using HTTP -import Distributed function Base.show(io::IO, s::SymbolsState) print(io, "SymbolsState([") @@ -226,11 +225,3 @@ has_embedded_pkgfiles(contents::AbstractString) = has_embedded_pkgfiles(nb::Pluto.Notebook) = read(nb.path, String) |> has_embedded_pkgfiles -""" -Log an error message if there are any running processes created by Distrubted, that were not shut down. -""" -function verify_no_running_processes() - if length(Distributed.procs()) != 1 - @error "Not all notebook processes were closed during tests!" Distributed.procs() - end -end diff --git a/test/packages/Basic.jl b/test/packages/Basic.jl index f64466899f..761c7dd1dd 100644 --- a/test/packages/Basic.jl +++ b/test/packages/Basic.jl @@ -6,7 +6,7 @@ using Pluto.WorkspaceManager: _merge_notebook_compiler_options, poll import Pluto: update_save_run!, update_run!, WorkspaceManager, ClientSession, ServerSession, Notebook, Cell, project_relative_path, SessionActions, load_notebook import Pluto.PkgUtils import Pluto.PkgCompat -import Distributed +import Malt # We have our own registry for these test! Take a look at https://github.com/JuliaPluto/PlutoPkgTestRegistry#readme for more info about the test packages and their dependencies. @@ -379,8 +379,7 @@ const pluto_test_registry_spec = Pkg.RegistrySpec(; end @testset "DrWatson cell" begin - 🍭 = ServerSession() - 🍭.options.evaluation.workspace_use_distributed = false + 🍭 = ServerSession() notebook = Notebook([ Cell("using Plots"), @@ -430,12 +429,13 @@ const pluto_test_registry_spec = Pkg.RegistrySpec(; end @testset "File format -- Forwards compat" begin - # Using Distributed, we will create a new Julia process in which we install Pluto 0.14.7 (before PlutoPkg). We run the new notebook file on the old Pluto. - p = Distributed.addprocs(1) |> first + # Using Malt, create a Julia process in which we install Pluto 0.14.7 (before PlutoPkg). + # Run the new notebook file on the old Pluto. + test_worker = Malt.Worker() @test post_pkg_notebook isa String - Distributed.remotecall_eval(Main, p, quote + Malt.remote_eval_wait(test_worker, quote path = tempname() write(path, $(post_pkg_notebook)) import Pkg @@ -444,35 +444,38 @@ const pluto_test_registry_spec = Pkg.RegistrySpec(; Pkg.UPDATED_REGISTRY_THIS_SESSION[] = true end - Pkg.activate(mktempdir()) + Pkg.activate(;temp=true) Pkg.add(Pkg.PackageSpec(;name="Pluto",version=v"0.14.7")) + # Distributed is required for old Pluto to work! + Pkg.add("Distributed") + import Pluto + @info Pluto.PLUTO_VERSION @assert Pluto.PLUTO_VERSION == v"0.14.7" + end) + @test Malt.remote_eval_fetch(test_worker, quote s = Pluto.ServerSession() - s.options.evaluation.workspace_use_distributed = false - nb = Pluto.SessionActions.open(s, path; run_async=false) - - nothing + nb.cells[2].errored == false end) # Cells that use Example will error because the package is not installed. - # @test Distributed.remotecall_eval(Main, p, quote + # @test Malt.remote_eval_fetch(test_worker, quote # nb.cells[1].errored == false # end) - @test Distributed.remotecall_eval(Main, p, quote - nb.cells[2].errored == false - end) - # @test Distributed.remotecall_eval(Main, p, quote + # @test Malt.remote_eval_fetch(test_worker, quote + # nb.cells[2].errored == false + # end) + # @test Malt.remote_eval_fetch(test_worker, quote # nb.cells[3].errored == false # end) - # @test Distributed.remotecall_eval(Main, p, quote + # @test Malt.remote_eval_fetch(test_worker, quote # nb.cells[3].output.body == "25" # end) - Distributed.rmprocs([p]) + Malt.stop(test_worker) end @testset "PkgUtils -- reset" begin diff --git a/test/runtests.jl b/test/runtests.jl index 3b3ce21a51..af377ef689 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -3,35 +3,22 @@ include("helpers.jl") # tests that start new processes: @timeit_include("compiletimes.jl") -verify_no_running_processes() if get(ENV, "PLUTO_TEST_ONLY_COMPILETIMES", nothing) == "true" print_timeroutput() exit(0) end @timeit_include("Events.jl") -verify_no_running_processes() @timeit_include("WorkspaceManager.jl") -verify_no_running_processes() @timeit_include("packages/Basic.jl") -verify_no_running_processes() @timeit_include("Bonds.jl") -verify_no_running_processes() @timeit_include("RichOutput.jl") -verify_no_running_processes() @timeit_include("React.jl") -verify_no_running_processes() @timeit_include("Dynamic.jl") -verify_no_running_processes() @timeit_include("MacroAnalysis.jl") -verify_no_running_processes() @timeit_include("Logging.jl") -verify_no_running_processes() @timeit_include("webserver.jl") -verify_no_running_processes() @timeit_include("Notebook.jl") -verify_no_running_processes() @timeit_include("Configuration.jl") -verify_no_running_processes() # tests that don't start new processes: @timeit_include("ReloadFromFile.jl") @@ -46,8 +33,6 @@ verify_no_running_processes() @timeit_include("Throttled.jl") @timeit_include("cell_disabling.jl") -verify_no_running_processes() - print_timeroutput() # TODO: test PlutoRunner functions like: diff --git a/test/webserver.jl b/test/webserver.jl index 9afbd3bdf1..1a643347be 100644 --- a/test/webserver.jl +++ b/test/webserver.jl @@ -26,7 +26,6 @@ using Pluto.WorkspaceManager: WorkspaceManager, poll options = Pluto.Configuration.from_flat_kwargs(; port, launch_browser=false, - workspace_use_distributed=true, require_secret_for_access=false, require_secret_for_open_links=false, base_url, @@ -69,7 +68,7 @@ end # without notebook at startup - options = Pluto.Configuration.from_flat_kwargs(; port, launch_browser=false, workspace_use_distributed=true, require_secret_for_access=false, require_secret_for_open_links=false) + options = Pluto.Configuration.from_flat_kwargs(; port, launch_browser=false, require_secret_for_access=false, require_secret_for_open_links=false) 🍭 = Pluto.ServerSession(; options) server_task = @async Pluto.run(🍭) @@ -91,7 +90,7 @@ end # right now, the notebook was only added to the session and assigned an ID. Let's wait for it to get a process: @test poll(60) do - haskey(WorkspaceManager.workspaces, notebook.notebook_id) + haskey(WorkspaceManager.active_workspaces, notebook.notebook_id) end sleep(1)