From 0d8f394e5945ba3010c8a1325a0c3e7762aa9266 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 25 Oct 2021 13:09:29 +0200 Subject: [PATCH 01/28] represent unresolved cells using a `Set{Cell}` instead of `Dict` --- src/analysis/Topology.jl | 2 +- src/analysis/TopologyUpdate.jl | 25 ++++++------- src/evaluation/Run.jl | 68 +++++++++++++++++++++++----------- 3 files changed, 59 insertions(+), 36 deletions(-) diff --git a/src/analysis/Topology.jl b/src/analysis/Topology.jl index cd3e1028ac..70f85fca17 100644 --- a/src/analysis/Topology.jl +++ b/src/analysis/Topology.jl @@ -30,7 +30,7 @@ Base.@kwdef struct NotebookTopology nodes::DefaultDict{Cell,ReactiveNode} = DefaultDict{Cell,ReactiveNode}(ReactiveNode) codes::DefaultDict{Cell,ExprAnalysisCache}=DefaultDict{Cell,ExprAnalysisCache}(ExprAnalysisCache) - unresolved_cells::Dict{Cell,SymbolsState} = Dict{Cell,SymbolsState}() + unresolved_cells::Set{Cell} = Set{Cell}() end diff --git a/src/analysis/TopologyUpdate.jl b/src/analysis/TopologyUpdate.jl index 1691537a3a..b454ee8cdc 100644 --- a/src/analysis/TopologyUpdate.jl +++ b/src/analysis/TopologyUpdate.jl @@ -6,30 +6,27 @@ function updated_topology(old_topology::NotebookTopology, notebook::Notebook, ce updated_codes = Dict{Cell,ExprAnalysisCache}() updated_nodes = Dict{Cell,ReactiveNode}() - unresolved_cells = Dict{Cell,SymbolsState}() + unresolved_cells = Set{Cell}() for cell in cells - if !( - haskey(old_topology.codes, cell) && - old_topology.codes[cell].code === cell.code - ) + if !(old_topology.codes[cell].code === cell.code) new_code = updated_codes[cell] = ExprAnalysisCache(notebook, cell) new_symstate = new_code.parsedcode |> ExpressionExplorer.try_compute_symbolreferences new_reactive_node = ReactiveNode(new_symstate) - if isempty(new_reactive_node.macrocalls) - updated_nodes[cell] = new_reactive_node - else - # The unresolved cells are the cells for wich we cannot create - # a ReactiveNode yet, because they contains macrocalls. - updated_nodes[cell] = new_reactive_node - unresolved_cells[cell] = new_symstate - end + updated_nodes[cell] = new_reactive_node + end + + new_reactive_node = get(updated_nodes, cell, old_topology.nodes[cell]) + if !isempty(new_reactive_node.macrocalls) + # The unresolved cells are the cells for wich we cannot create + # a ReactiveNode yet, because they contains macrocalls. + push!(unresolved_cells, cell) end end new_codes = merge(old_topology.codes, updated_codes) new_nodes = merge(old_topology.nodes, updated_nodes) - new_unresolved_cells = merge(old_topology.unresolved_cells, unresolved_cells) + new_unresolved_cells = union(old_topology.unresolved_cells, unresolved_cells) for removed_cell in setdiff(keys(old_topology.nodes), notebook.cells) delete!(new_nodes, removed_cell) diff --git a/src/evaluation/Run.jl b/src/evaluation/Run.jl index b51d1856fe..9cf2bc5f33 100644 --- a/src/evaluation/Run.jl +++ b/src/evaluation/Run.jl @@ -15,7 +15,7 @@ function run_reactive!(session::ServerSession, notebook::Notebook, old_topology: if !is_resolved(new_topology) unresolved_topology = new_topology - new_topology = notebook.topology = resolve_topology(session, notebook, unresolved_topology, old_workspace_name) + new_topology = notebook.topology = resolve_topology(session, notebook, unresolved_topology, old_workspace_name; current_roots=roots) # update cache and save notebook because the dependencies might have changed after expanding macros update_dependency_cache!(notebook) @@ -227,6 +227,9 @@ end collect_implicit_usings(topology::NotebookTopology, cell::Cell) = ExpressionExplorer.collect_implicit_usings(topology.codes[cell].module_usings_imports) +"Returns the set of macros names defined by this cell" +defined_macros(topology::NotebookTopology, cell::Cell) = filter(is_macro_identifier, topology.nodes[cell].funcdefs_without_signatures) + "Tells whether or not a cell can 'unlock' the resolution of other cells" function can_help_resolve_cells(topology::NotebookTopology, cell::Cell) cell_code = topology.codes[cell] @@ -236,8 +239,22 @@ function can_help_resolve_cells(topology::NotebookTopology, cell::Cell) any(is_macro_identifier, cell_node.funcdefs_without_signatures) end -"We still have 'unresolved' macrocalls, use the current workspace to do macro-expansions" -function resolve_topology(session::ServerSession, notebook::Notebook, unresolved_topology::NotebookTopology, old_workspace_name::Symbol) +"""We still have 'unresolved' macrocalls, use the current and maybe previous workspace to do macro-expansions. + +You can optionally specify the roots for the current reactive run. If a cell macro contains only macros that will +be re-defined during this reactive run, we don't expand yet and expect the `can_help_resolve_cells` function above +to be true for the cell defining the macro, triggering a new topology resolution without needing to fallback to the +previous workspace. +""" +function resolve_topology( + session::ServerSession, + notebook::Notebook, + unresolved_topology::NotebookTopology, + old_workspace_name::Symbol; + current_roots::Vector{Cell}=Cell[], +)::NotebookTopology + + sn = (session, notebook) function macroexpand_cell(cell) @@ -251,34 +268,41 @@ function resolve_topology(session::ServerSession, notebook::Notebook, unresolved res end - function analyze_macrocell(cell::Cell, current_symstate) + # nothing means that the expansion has failed + function analyze_macrocell(cell::Cell) if unresolved_topology.nodes[cell].macrocalls βŠ† ExpressionExplorer.can_macroexpand - return current_symstate, true + return nothing end result = macroexpand_cell(cell) if result isa Exception - # if expansion failed, we use the "shallow" symbols state @debug "Expansion failed" err=result - current_symstate, false + nothing else # otherwise, we use the expanded expression + the list of macrocalls - expanded_symbols_state = ExpressionExplorer.try_compute_symbolreferences(result) - expanded_symbols_state, true + expanded_node = ExpressionExplorer.try_compute_symbolreferences(result) |> ReactiveNode + expanded_node end end + run_defined_macros = mapreduce(c -> defined_macros(unresolved_topology, c), union!, current_roots; init=Set{Symbol}()) + # create new node & new codes for macrocalled cells new_nodes = Dict{Cell,ReactiveNode}() - still_unresolved_nodes = Dict{Cell,SymbolsState}() - for (cell, current_symstate) in unresolved_topology.unresolved_cells - (new_symstate, succeeded) = analyze_macrocell(cell, current_symstate) - if succeeded - new_node = ReactiveNode(new_symstate) + still_unresolved_nodes = Set{Cell}() + for cell in unresolved_topology.unresolved_cells + if unresolved_topology.nodes[cell].macrocalls βŠ† run_defined_macros + # Do not try to expand if a newer version of the macro is also scheduled to run in the + # current run. The recursive reactive runs will take care of it. + push!(still_unresolved_nodes, cell) + end + + new_node = analyze_macrocell(cell) + if new_node !== nothing union!(new_node.macrocalls, unresolved_topology.nodes[cell].macrocalls) union!(new_node.references, new_node.macrocalls) new_nodes[cell] = new_node else - still_unresolved_nodes[cell] = current_symstate + push!(still_unresolved_nodes, cell) end end @@ -287,17 +311,19 @@ function resolve_topology(session::ServerSession, notebook::Notebook, unresolved NotebookTopology(nodes=all_nodes, codes=unresolved_topology.codes, unresolved_cells=still_unresolved_nodes) end -function static_macroexpand(topology::NotebookTopology, cell::Cell, old_symstate) - new_symstate = ExpressionExplorer.maybe_macroexpand(topology.codes[cell].parsedcode; recursive=true) |> - ExpressionExplorer.try_compute_symbolreferences - union!(new_symstate.macrocalls, old_symstate.macrocalls) +function static_macroexpand(topology::NotebookTopology, cell::Cell) + cell_node = topology.nodes[cell].macrocalls + + new_node = ExpressionExplorer.maybe_macroexpand(topology.codes[cell].parsedcode; recursive=true) |> + ExpressionExplorer.try_compute_symbolreferences |> ReactiveNode + union!(new_node.macrocalls, cell_node.macrocalls) - ReactiveNode(new_symstate) + new_node end "The same as `resolve_topology` but does not require custom code execution, only works with a few `Base` & `PlutoRunner` macros" function static_resolve_topology(topology::NotebookTopology) - new_nodes = Dict{Cell,ReactiveNode}(cell => static_macroexpand(topology, cell, symstate) for (cell, symstate) in topology.unresolved_cells) + new_nodes = Dict{Cell,ReactiveNode}(cell => static_macroexpand(topology, cell) for cell in topology.unresolved_cells) all_nodes = merge(topology.nodes, new_nodes) NotebookTopology(nodes=all_nodes, codes=topology.codes, unresolved_cells=topology.unresolved_cells) From a4c6a3d0a0c3f1b9cf11fe9a9a227bdb4357b229 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 25 Oct 2021 13:28:44 +0200 Subject: [PATCH 02/28] add several broken macro tests and some fixed too --- test/MacroAnalysis.jl | 111 ++++++++++++++++++++++++++++++++++++++++++ test/helpers.jl | 4 +- 2 files changed, 113 insertions(+), 2 deletions(-) diff --git a/test/MacroAnalysis.jl b/test/MacroAnalysis.jl index 2f0454fdbe..fd593c3266 100644 --- a/test/MacroAnalysis.jl +++ b/test/MacroAnalysis.jl @@ -280,6 +280,117 @@ import Pluto: PlutoRunner, Notebook, WorkspaceManager, Cell, ServerSession, Clie @test cell(1).output.body == "42" end + @testset "Redefines macro with new SymbolsState" begin + 🍭.options.evaluation.workspace_use_distributed = true + + notebook = Notebook(Cell.([ + "@b x", + raw"""macro b(sym) + esc(:($sym = 42)) + end""", + "x", + "y", + ])) + cell(idx) = notebook.cells[idx] + update_run!(🍭, notebook, notebook.cells) + + @test cell(3).output.body == "42" + @test cell(4).errored == true + + setcode(cell(2), """macro b(_) + esc(:(y = 42)) + end""") + update_run!(🍭, notebook, cell(2)) + + # Cell 4 is not re-executed because the ReactiveNode + # for @b(x) has not changed. + @test_broken cell(4).output.body == "42" + @test cell(3).errored == true + + WorkspaceManager.unmake_workspace((🍭, notebook)) + + notebook = Notebook(Cell.([ + "@b x", + raw"""macro b(sym) + esc(:($sym = 42)) + end""", + "x", + "y", + ])) + update_run!(🍭, notebook, notebook.cells) + + @test cell(3).output.body == "42" + @test cell(4).errored == true + + setcode(cell(2), """macro b(_) + esc(:(y = 42)) + end""") + update_run!(🍭, notebook, [cell(1), cell(2)]) + + # Cell 4 is executed even because cell(1) is in the root + # of the reactive run because the expansion is done with the new version + # of the macro in the new workspace because of the current_roots parameter of `resolve_topology`. + # See Run.jl#resolve_topology. + @test cell(4).output.body == "42" + @test cell(3).errored == true + + WorkspaceManager.unmake_workspace((🍭, notebook)) + 🍭.options.evaluation.workspace_use_distributed = false + end + + @testset "Reactive macro update does not invalidate the macro calls" begin + notebook = Notebook(Cell.([ + raw"""macro b(sym) + if z > 40 + esc(:($sym = $z)) + else + esc(:(y = $z)) + end + end""", + "z = 42", + "@b(x)", + "x", + "y", + ])) + cell(idx) = notebook.cells[idx] + update_run!(🍭, notebook, notebook.cells) + + @test cell(1) |> noerror + @test cell(2) |> noerror + @test cell(3) |> noerror + @test cell(4) |> noerror + @test cell(5).errored == true + + setcode(cell(2), "z = 39") + + # running only 2, running all cells here works however + update_run!(🍭, notebook, cell(2)) + + @test cell(1) |> noerror + @test cell(2) |> noerror + @test cell(3) |> noerror + @test cell(4).errored == true + + # cell 5 should re-run because all cells calling @b should be invalidated + # at the reactive node level this is not yet the case + @test_broken noerror(cell(5); verbose=false) + end + + @testset "Cell failing first not re-run?" begin + notebook = Notebook(Cell.([ + "x", + "@b x", + raw"macro b(sym) esc(:($sym = 42))", + ])) + update_run!(🍭, notebook, notebook.cells) + + # CELL 1 "x" was run first and failed because the definition + # of x was not yet found. However, it was not run re-run when the definition of + # x ("@b(x)") was run. Should it? Maybe set a higher precedence to cells that define + # macros inside the notebook. + @test_broken noerror(notebook.cells[1]; verbose=false) + end + @testset "@a defines @b initial loading" begin notebook = Notebook(Cell.([ "x", diff --git a/test/helpers.jl b/test/helpers.jl index 4ebaa4f049..5d3f64d9d3 100644 --- a/test/helpers.jl +++ b/test/helpers.jl @@ -116,8 +116,8 @@ function setcode(cell, newcode) cell.code = newcode end -function noerror(cell) - if cell.errored +function noerror(cell; verbose=true) + if cell.errored && verbose @show cell.output.body end !cell.errored From 5dc100a0eba0885e000cef5a0c2bfb5b5a73b786 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 25 Oct 2021 14:05:27 +0200 Subject: [PATCH 03/28] unresolve dependencies when defining a new macro --- src/analysis/Topology.jl | 4 +++ src/evaluation/Run.jl | 10 +++++- test/MacroAnalysis.jl | 69 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 1 deletion(-) diff --git a/src/analysis/Topology.jl b/src/analysis/Topology.jl index 70f85fca17..5336426c91 100644 --- a/src/analysis/Topology.jl +++ b/src/analysis/Topology.jl @@ -36,6 +36,10 @@ end is_resolved(topology::NotebookTopology) = isempty(topology.unresolved_cells) +function set_unresolved(topology::NotebookTopology, unresolved_cells::Vector{Cell}) + NotebookTopology(nodes=topology.nodes, codes=topology.codes, unresolved_cells=union(topology.unresolved_cells, unresolved_cells)) +end + DefaultDict{K,V}(default::Union{Function,DataType}) where {K,V} = DefaultDict{K,V}(default, Dict{K,V}()) function Base.getindex(aid::DefaultDict{K,V}, key::K)::V where {K,V} diff --git a/src/evaluation/Run.jl b/src/evaluation/Run.jl index 9cf2bc5f33..a1fbc1a7b8 100644 --- a/src/evaluation/Run.jl +++ b/src/evaluation/Run.jl @@ -122,8 +122,16 @@ function run_reactive!(session::ServerSession, notebook::Notebook, old_topology: implicit_usings = collect_implicit_usings(new_topology, cell) if !is_resolved(new_topology) && can_help_resolve_cells(new_topology, cell) + defined_macros_in_cell = defined_macros(new_topology, cell) |> Set{Symbol} + + # Also set unresolved the downstream cells using the defined macros + if !isempty(defined_macros_in_cell) + new_topology = set_unresolved(new_topology, where_referenced(notebook, new_topology, defined_macros_in_cell)) + end + notebook.topology = new_new_topology = resolve_topology(session, notebook, new_topology, old_workspace_name) + if !isempty(implicit_usings) new_soft_definitions = WorkspaceManager.collect_soft_definitions((session, notebook), implicit_usings) notebook.topology = new_new_topology = with_new_soft_definitions(new_new_topology, cell, new_soft_definitions) @@ -234,7 +242,7 @@ defined_macros(topology::NotebookTopology, cell::Cell) = filter(is_macro_identif function can_help_resolve_cells(topology::NotebookTopology, cell::Cell) cell_code = topology.codes[cell] cell_node = topology.nodes[cell] - !isempty(cell_code.module_usings_imports.imports) || + !isempty(cell_code.module_usings_imports.imports) || # <-- TODO(paul): check explicitely for `import Pkg: @macro` instead of any imports !isempty(cell_code.module_usings_imports.usings) || any(is_macro_identifier, cell_node.funcdefs_without_signatures) end diff --git a/test/MacroAnalysis.jl b/test/MacroAnalysis.jl index fd593c3266..42d4d7f25e 100644 --- a/test/MacroAnalysis.jl +++ b/test/MacroAnalysis.jl @@ -376,6 +376,75 @@ import Pluto: PlutoRunner, Notebook, WorkspaceManager, Cell, ServerSession, Clie @test_broken noerror(cell(5); verbose=false) end + @testset "Explicitely running macrocalls updates the reactive node" begin + notebook = Notebook(Cell.([ + "@b()", + "ref = Ref{Int}(0)", + raw"""macro b() + ex = if iseven(ref[]) + :(x = 10) + else + :(y = 10) + end |> esc + ref[] += 1 + ex + end""", + "x", + "y", + ])) + cell(i) = notebook.cells[i] + update_run!(🍭, notebook, notebook.cells) + + @test cell(1) |> noerror + @test cell(2) |> noerror + @test cell(3) |> noerror + @test cell(4) |> noerror + @test cell(5).errored == true + + update_run!(🍭, notebook, cell(1)) + + @test cell(4).errored == true + @test cell(5) |> noerror + end + + @testset "Implicitely running macrocalls updates the reactive node" begin + notebook = Notebook(Cell.([ + "updater; @b()", + "ref = Ref{Int}(0)", + raw"""macro b() + ex = if iseven(ref[]) + :(x = 10) + else + :(y = 10) + end |> esc + ref[] += 1 + ex + end""", + "x", + "y", + "updater = 1", + ])) + cell(i) = notebook.cells[i] + update_run!(🍭, notebook, notebook.cells) + + @test cell(1) |> noerror + @test cell(2) |> noerror + @test cell(3) |> noerror + @test cell(4) |> noerror + @test cell(5).errored == true + @test cell(6) |> noerror + + setcode(cell(6), "updater = 2") + update_run!(🍭, notebook, cell(6)) + + @test cell(4).errored == true + + # Since the run of `@b()` was not explicit, + # the reactive node of cell(1) was not updated :'( + @test_broken noerror(cell(5); verbose=false) + end + + @testset "Cell failing first not re-run?" begin notebook = Notebook(Cell.([ "x", From e925ef14515f08d5b90abbecfc0d5056341bbb07 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 25 Oct 2021 14:13:49 +0200 Subject: [PATCH 04/28] not distributed --- test/MacroAnalysis.jl | 7 ------- 1 file changed, 7 deletions(-) diff --git a/test/MacroAnalysis.jl b/test/MacroAnalysis.jl index 42d4d7f25e..5e5ad1ad98 100644 --- a/test/MacroAnalysis.jl +++ b/test/MacroAnalysis.jl @@ -281,8 +281,6 @@ import Pluto: PlutoRunner, Notebook, WorkspaceManager, Cell, ServerSession, Clie end @testset "Redefines macro with new SymbolsState" begin - 🍭.options.evaluation.workspace_use_distributed = true - notebook = Notebook(Cell.([ "@b x", raw"""macro b(sym) @@ -307,8 +305,6 @@ import Pluto: PlutoRunner, Notebook, WorkspaceManager, Cell, ServerSession, Clie @test_broken cell(4).output.body == "42" @test cell(3).errored == true - WorkspaceManager.unmake_workspace((🍭, notebook)) - notebook = Notebook(Cell.([ "@b x", raw"""macro b(sym) @@ -333,9 +329,6 @@ import Pluto: PlutoRunner, Notebook, WorkspaceManager, Cell, ServerSession, Clie # See Run.jl#resolve_topology. @test cell(4).output.body == "42" @test cell(3).errored == true - - WorkspaceManager.unmake_workspace((🍭, notebook)) - 🍭.options.evaluation.workspace_use_distributed = false end @testset "Reactive macro update does not invalidate the macro calls" begin From 51c5a84a2d5eab215c9f23b9d76dd1b80a7048dc Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 25 Oct 2021 14:29:10 +0200 Subject: [PATCH 05/28] fix static_resolve --- src/evaluation/Run.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/evaluation/Run.jl b/src/evaluation/Run.jl index a1fbc1a7b8..4ddb34b1d6 100644 --- a/src/evaluation/Run.jl +++ b/src/evaluation/Run.jl @@ -319,12 +319,13 @@ function resolve_topology( NotebookTopology(nodes=all_nodes, codes=unresolved_topology.codes, unresolved_cells=still_unresolved_nodes) end +"""Tries to add information about macro calls without running any code, using knowledge about common macros. +So, the resulting reactive nodes may not be absolutely accurate. If you can run code in a session, use `resolve_topology` instead. +""" function static_macroexpand(topology::NotebookTopology, cell::Cell) - cell_node = topology.nodes[cell].macrocalls - new_node = ExpressionExplorer.maybe_macroexpand(topology.codes[cell].parsedcode; recursive=true) |> ExpressionExplorer.try_compute_symbolreferences |> ReactiveNode - union!(new_node.macrocalls, cell_node.macrocalls) + union!(new_node.macrocalls, topology.nodes[cell].macrocalls) new_node end From 598575a2e8da5bd8ae082113e37f8b7c4762c0ff Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 25 Oct 2021 16:47:48 +0200 Subject: [PATCH 06/28] fix test --- test/MacroAnalysis.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/MacroAnalysis.jl b/test/MacroAnalysis.jl index 5e5ad1ad98..ffc9b6b57e 100644 --- a/test/MacroAnalysis.jl +++ b/test/MacroAnalysis.jl @@ -442,7 +442,7 @@ import Pluto: PlutoRunner, Notebook, WorkspaceManager, Cell, ServerSession, Clie notebook = Notebook(Cell.([ "x", "@b x", - raw"macro b(sym) esc(:($sym = 42))", + raw"macro b(sym) esc(:($sym = 42)) end", ])) update_run!(🍭, notebook, notebook.cells) From a5865c39584cc1172a78ce4c34025afcec4ec1dd Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 25 Oct 2021 17:23:30 +0200 Subject: [PATCH 07/28] test #1591 --- src/analysis/TopologyUpdate.jl | 9 ++++++--- test/MacroAnalysis.jl | 27 +++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/analysis/TopologyUpdate.jl b/src/analysis/TopologyUpdate.jl index b454ee8cdc..db3bc76d28 100644 --- a/src/analysis/TopologyUpdate.jl +++ b/src/analysis/TopologyUpdate.jl @@ -6,9 +6,9 @@ function updated_topology(old_topology::NotebookTopology, notebook::Notebook, ce updated_codes = Dict{Cell,ExprAnalysisCache}() updated_nodes = Dict{Cell,ReactiveNode}() - unresolved_cells = Set{Cell}() + unresolved_cells = copy(old_topology.unresolved_cells) for cell in cells - if !(old_topology.codes[cell].code === cell.code) + if old_topology.codes[cell].code !== cell.code new_code = updated_codes[cell] = ExprAnalysisCache(notebook, cell) new_symstate = new_code.parsedcode |> ExpressionExplorer.try_compute_symbolreferences @@ -22,11 +22,14 @@ function updated_topology(old_topology::NotebookTopology, notebook::Notebook, ce # The unresolved cells are the cells for wich we cannot create # a ReactiveNode yet, because they contains macrocalls. push!(unresolved_cells, cell) + else + pop!(unresolved_cells, cell, nothing) end end new_codes = merge(old_topology.codes, updated_codes) new_nodes = merge(old_topology.nodes, updated_nodes) - new_unresolved_cells = union(old_topology.unresolved_cells, unresolved_cells) + # new_unresolved_cells = union(old_topology.unresolved_cells, unresolved_cells) + new_unresolved_cells = unresolved_cells for removed_cell in setdiff(keys(old_topology.nodes), notebook.cells) delete!(new_nodes, removed_cell) diff --git a/test/MacroAnalysis.jl b/test/MacroAnalysis.jl index ffc9b6b57e..1e90b4a947 100644 --- a/test/MacroAnalysis.jl +++ b/test/MacroAnalysis.jl @@ -437,6 +437,33 @@ import Pluto: PlutoRunner, Notebook, WorkspaceManager, Cell, ServerSession, Clie @test_broken noerror(cell(5); verbose=false) end + @testset "Weird behavior" begin + # https://github.com/fonsp/Pluto.jl/issues/1591 + + notebook = Notebook(Cell.([ + "macro huh(_) throw(\"Fail!\") end", + "huh(e) = e", + "@huh(z)", + "z = 101010", + ])) + cell(idx) = notebook.cells[idx] + update_run!(🍭, notebook, notebook.cells) + + @test cell(3).errored == true + + setcode(cell(3), "huh(z)") + update_run!(🍭, notebook, cell(3)) + + @test cell(3) |> noerror + @test cell(3).output.body == "101010" + + setcode(cell(4), "z = 1234") + update_run!(🍭, notebook, cell(4)) + + @test cell(3) |> noerror + @test cell(3).output.body == "1234" + end + @testset "Cell failing first not re-run?" begin notebook = Notebook(Cell.([ From 0116f953ff2d760dccde72c27a26fe2a1fab9fe9 Mon Sep 17 00:00:00 2001 From: Paul Date: Tue, 26 Oct 2021 13:33:14 +0200 Subject: [PATCH 08/28] report real timings for macros --- src/runner/PlutoRunner.jl | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/src/runner/PlutoRunner.jl b/src/runner/PlutoRunner.jl index bccd1fb64c..592cb4a9d8 100644 --- a/src/runner/PlutoRunner.jl +++ b/src/runner/PlutoRunner.jl @@ -26,7 +26,7 @@ export @bind MimedOutput = Tuple{Union{String,Vector{UInt8},Dict{Symbol,Any}},MIME} const ObjectID = typeof(objectid("hello computer")) const ObjectDimPair = Tuple{ObjectID,Int64} -const ExpandedCallCells = Dict{UUID,Expr}() +const ExpandedCallCells = Dict{UUID,NamedTuple{(:expr,:runtime),Tuple{Expr,UInt64}}}() @@ -178,8 +178,11 @@ sanitize_value(other) = sanitize_expr(other) function try_macroexpand(mod, cell_uuid, expr) try + elapsed_ns = time_ns() expanded_expr = macroexpand(mod, expr) - ExpandedCallCells[cell_uuid] = no_workspace_ref(expanded_expr) + elapsed_ns = time_ns() - elapsed_ns + + ExpandedCallCells[cell_uuid] = (;expr=no_workspace_ref(expanded_expr), runtime=elapsed_ns) return sanitize_expr(expanded_expr) catch e @@ -323,16 +326,16 @@ function run_inside_trycatch(m::Module, f::Union{Expr,Function}, return_proof::R end end - -visit_expand(_, other) = other -function visit_expand(current_module::Module, expr::Expr) - if expr.head == :macrocall - no_workspace_ref(macroexpand(current_module, expr), nameof(current_module)) - else - Expr(expr.head, map(arg -> visit_expand(current_module, arg), expr.args)...) - end +function timed_visit_expand(current_module::Module, expr::Expr)::Tuple{Expr,UInt64} + elapsed = time_ns() + expanded_expr = no_workspace_ref(macroexpand(current_module, expr), nameof(current_module)) # visit_expand(current_module, expr) + elapsed = time_ns() - elapsed + (expanded_expr, elapsed) end +add_runtimes(::Nothing, ::UInt64) = nothing +add_runtimes(a::UInt64, b::UInt64) = a+b + """ Run the given expression in the current workspace module. If the third argument is `nothing`, then the expression will be `Core.eval`ed. The result and runtime are stored inside [`cell_results`](@ref) and [`cell_runtimes`](@ref). @@ -345,22 +348,28 @@ function run_expression(m::Module, expr::Any, cell_id::UUID, function_wrapped_in cell_published_objects[cell_id] = Dict{String,Any}() result, runtime = if function_wrapped_info === nothing - expr = pop!(ExpandedCallCells, cell_id, expr) + expanded = pop!(ExpandedCallCells, cell_id, (;expr,runtime=zero(UInt64))) + expr = expanded.expr + expansion_runtime = expanded.runtime + proof = ReturnProof() # Note: fix for https://github.com/fonsp/Pluto.jl/issues/1112 if contains_user_defined_macrocalls try - expr = visit_expand(m, expr) + (expr, other_expansion_runtime) = timed_visit_expand(m, expr) + expansion_runtime = max(expansion_runtime, other_expansion_runtime) wrapped = timed_expr(expr, proof) - run_inside_trycatch(m, wrapped, proof) + ans, runtime = run_inside_trycatch(m, wrapped, proof) + (ans, add_runtimes(runtime, expansion_runtime)) catch ex bt = stacktrace(catch_backtrace()) (CapturedException(ex, bt), nothing) end else wrapped = timed_expr(expr, proof) - run_inside_trycatch(m, wrapped, proof) + ans, runtime = run_inside_trycatch(m, wrapped, proof) + (ans, add_runtimes(runtime, expansion_runtime)) end else key = expr_hash(expr) From 19f6f6e264fb571ac0bcacf847615d55816ffbbc Mon Sep 17 00:00:00 2001 From: Paul Date: Tue, 26 Oct 2021 16:14:31 +0200 Subject: [PATCH 09/28] wrap macro calls in functions --- src/analysis/Topology.jl | 11 ++++- src/analysis/TopologyUpdate.jl | 12 +++--- src/evaluation/Run.jl | 34 +++++++++------ src/evaluation/WorkspaceManager.jl | 15 +++++-- src/runner/PlutoRunner.jl | 15 ++++--- test/MacroAnalysis.jl | 67 ++++++++++++++++++++++++------ 6 files changed, 112 insertions(+), 42 deletions(-) diff --git a/src/analysis/Topology.jl b/src/analysis/Topology.jl index 5336426c91..cea0c7ce1f 100644 --- a/src/analysis/Topology.jl +++ b/src/analysis/Topology.jl @@ -4,8 +4,9 @@ import .ExpressionExplorer: UsingsImports, SymbolsState Base.@kwdef struct ExprAnalysisCache code::String="" parsedcode::Expr=Expr(:toplevel, LineNumberNode(1), Expr(:block)) - module_usings_imports::UsingsImports = UsingsImports() + module_usings_imports::UsingsImports = UsingsImports() function_wrapped::Bool=false + forced_expr_id::Union{PlutoRunner.ObjectID,Nothing}=nothing end ExprAnalysisCache(notebook, cell::Cell) = let @@ -18,6 +19,11 @@ ExprAnalysisCache(notebook, cell::Cell) = let ) end +function ExprAnalysisCache(old_cache::ExprAnalysisCache; new_properties...) + properties = Dict{Symbol,Any}(field => getproperty(old_cache, field) for field in fieldnames(ExprAnalysisCache)) + merge!(properties, Dict{Symbol,Any}(new_properties)) + ExprAnalysisCache(;properties...) +end struct DefaultDict{K,V} <: AbstractDict{K,V} default::Union{Function,DataType} @@ -37,7 +43,8 @@ end is_resolved(topology::NotebookTopology) = isempty(topology.unresolved_cells) function set_unresolved(topology::NotebookTopology, unresolved_cells::Vector{Cell}) - NotebookTopology(nodes=topology.nodes, codes=topology.codes, unresolved_cells=union(topology.unresolved_cells, unresolved_cells)) + codes = Dict{Cell,ExprAnalysisCache}(cell => ExprAnalysisCache(topology.codes[cell]; function_wrapped=false, forced_expr_id=nothing) for cell in unresolved_cells) + NotebookTopology(nodes=topology.nodes, codes=merge(topology.codes, codes), unresolved_cells=union(topology.unresolved_cells, unresolved_cells)) end DefaultDict{K,V}(default::Union{Function,DataType}) where {K,V} = DefaultDict{K,V}(default, Dict{K,V}()) diff --git a/src/analysis/TopologyUpdate.jl b/src/analysis/TopologyUpdate.jl index db3bc76d28..2378396eb7 100644 --- a/src/analysis/TopologyUpdate.jl +++ b/src/analysis/TopologyUpdate.jl @@ -8,13 +8,17 @@ function updated_topology(old_topology::NotebookTopology, notebook::Notebook, ce updated_nodes = Dict{Cell,ReactiveNode}() unresolved_cells = copy(old_topology.unresolved_cells) for cell in cells - if old_topology.codes[cell].code !== cell.code + old_code = old_topology.codes[cell] + if old_code.code !== cell.code new_code = updated_codes[cell] = ExprAnalysisCache(notebook, cell) new_symstate = new_code.parsedcode |> ExpressionExplorer.try_compute_symbolreferences new_reactive_node = ReactiveNode(new_symstate) updated_nodes[cell] = new_reactive_node + elseif old_code.forced_expr_id !== nothing + # reset computer code + updated_codes[cell] = ExprAnalysisCache(old_code; forced_expr_id=nothing, function_wrapped=false) end new_reactive_node = get(updated_nodes, cell, old_topology.nodes[cell]) @@ -28,14 +32,12 @@ function updated_topology(old_topology::NotebookTopology, notebook::Notebook, ce end new_codes = merge(old_topology.codes, updated_codes) new_nodes = merge(old_topology.nodes, updated_nodes) - # new_unresolved_cells = union(old_topology.unresolved_cells, unresolved_cells) - new_unresolved_cells = unresolved_cells for removed_cell in setdiff(keys(old_topology.nodes), notebook.cells) delete!(new_nodes, removed_cell) delete!(new_codes, removed_cell) - delete!(new_unresolved_cells, removed_cell) + delete!(unresolved_cells, removed_cell) end - NotebookTopology(nodes=new_nodes, codes=new_codes, unresolved_cells=new_unresolved_cells) + NotebookTopology(nodes=new_nodes, codes=new_codes, unresolved_cells=unresolved_cells) end diff --git a/src/evaluation/Run.jl b/src/evaluation/Run.jl index 4ddb34b1d6..7b1c53f0d4 100644 --- a/src/evaluation/Run.jl +++ b/src/evaluation/Run.jl @@ -120,15 +120,15 @@ function run_reactive!(session::ServerSession, notebook::Notebook, old_topology: cell.running = false - implicit_usings = collect_implicit_usings(new_topology, cell) - if !is_resolved(new_topology) && can_help_resolve_cells(new_topology, cell) - defined_macros_in_cell = defined_macros(new_topology, cell) |> Set{Symbol} + defined_macros_in_cell = defined_macros(new_topology, cell) |> Set{Symbol} - # Also set unresolved the downstream cells using the defined macros - if !isempty(defined_macros_in_cell) - new_topology = set_unresolved(new_topology, where_referenced(notebook, new_topology, defined_macros_in_cell)) - end + # Also set unresolved the downstream cells using the defined macros + if !isempty(defined_macros_in_cell) + new_topology = set_unresolved(new_topology, where_referenced(notebook, new_topology, defined_macros_in_cell)) + end + implicit_usings = collect_implicit_usings(new_topology, cell) + if !is_resolved(new_topology) && can_help_resolve_cells(new_topology, cell) notebook.topology = new_new_topology = resolve_topology(session, notebook, new_topology, old_workspace_name) @@ -200,6 +200,7 @@ function run_single!(session_notebook::Union{Tuple{ServerSession,Notebook},Works cell.cell_id, ends_with_semicolon(cell.code), expr_cache.function_wrapped ? (filter(!is_joined_funcname, reactive_node.references), reactive_node.definitions) : nothing, + expr_cache.forced_expr_id, cell.cell_dependencies.contains_user_defined_macrocalls, ) set_output!(cell, run, expr_cache; persist_js_state=persist_js_state) @@ -287,8 +288,10 @@ function resolve_topology( @debug "Expansion failed" err=result nothing else # otherwise, we use the expanded expression + the list of macrocalls - expanded_node = ExpressionExplorer.try_compute_symbolreferences(result) |> ReactiveNode - expanded_node + (expr, computer_id) = result + expanded_node = ExpressionExplorer.try_compute_symbolreferences(expr) |> ReactiveNode + function_wrapped = ExpressionExplorer.can_be_function_wrapped(expr) + (expanded_node, function_wrapped, computer_id) end end @@ -296,7 +299,9 @@ function resolve_topology( # create new node & new codes for macrocalled cells new_nodes = Dict{Cell,ReactiveNode}() + new_codes = Dict{Cell,ExprAnalysisCache}() still_unresolved_nodes = Set{Cell}() + for cell in unresolved_topology.unresolved_cells if unresolved_topology.nodes[cell].macrocalls βŠ† run_defined_macros # Do not try to expand if a newer version of the macro is also scheduled to run in the @@ -304,19 +309,24 @@ function resolve_topology( push!(still_unresolved_nodes, cell) end - new_node = analyze_macrocell(cell) - if new_node !== nothing + result = analyze_macrocell(cell) + if result !== nothing + (new_node, function_wrapped, forced_expr_id) = result union!(new_node.macrocalls, unresolved_topology.nodes[cell].macrocalls) union!(new_node.references, new_node.macrocalls) new_nodes[cell] = new_node + + # set function_wrapped to the function wrapped analysis of the expanded expression. + new_codes[cell] = ExprAnalysisCache(unresolved_topology.codes[cell]; forced_expr_id, function_wrapped) else push!(still_unresolved_nodes, cell) end end all_nodes = merge(unresolved_topology.nodes, new_nodes) + all_codes = merge(unresolved_topology.codes, new_codes) - NotebookTopology(nodes=all_nodes, codes=unresolved_topology.codes, unresolved_cells=still_unresolved_nodes) + NotebookTopology(nodes=all_nodes, codes=all_codes, unresolved_cells=still_unresolved_nodes) end """Tries to add information about macro calls without running any code, using knowledge about common macros. diff --git a/src/evaluation/WorkspaceManager.jl b/src/evaluation/WorkspaceManager.jl index ba8497fcff..b513f4ebff 100644 --- a/src/evaluation/WorkspaceManager.jl +++ b/src/evaluation/WorkspaceManager.jl @@ -249,7 +249,16 @@ end "Evaluate expression inside the workspace - output is fetched and formatted, errors are caught and formatted. Returns formatted output and error flags. `expr` has to satisfy `ExpressionExplorer.is_toplevel_expr`." -function eval_format_fetch_in_workspace(session_notebook::Union{SN,Workspace}, expr::Expr, cell_id::UUID, ends_with_semicolon::Bool=false, function_wrapped_info::Union{Nothing,Tuple}=nothing, contains_user_defined_macrocalls::Bool=false)::NamedTuple{(:output_formatted, :errored, :interrupted, :process_exited, :runtime, :published_objects),Tuple{PlutoRunner.MimedOutput,Bool,Bool,Bool,Union{UInt64,Nothing},Dict{String,Any}}} +function eval_format_fetch_in_workspace( + session_notebook::Union{SN,Workspace}, + expr::Expr, + cell_id::UUID, + ends_with_semicolon::Bool=false, + function_wrapped_info::Union{Nothing,Tuple}=nothing, + forced_expr_id::Union{PlutoRunner.ObjectID,Nothing}=nothing, + contains_user_defined_macrocalls::Bool=false +)::NamedTuple{(:output_formatted, :errored, :interrupted, :process_exited, :runtime, :published_objects),Tuple{PlutoRunner.MimedOutput,Bool,Bool,Bool,Union{UInt64,Nothing},Dict{String,Any}}} + workspace = get_workspace(session_notebook) # if multiple notebooks run on the same process, then we need to `cd` between the different notebook paths @@ -271,6 +280,7 @@ function eval_format_fetch_in_workspace(session_notebook::Union{SN,Workspace}, e $(QuoteNode(expr)), $cell_id, $function_wrapped_info, + $forced_expr_id, $contains_user_defined_macrocalls, ))) put!(workspace.dowork_token) @@ -330,8 +340,7 @@ function macroexpand_in_workspace(session_notebook::Union{SN,Workspace}, macroca PlutoRunner.try_macroexpand($(module_name), $(cell_uuid), $(macrocall |> QuoteNode)) end try - result = Distributed.remotecall_eval(Main, workspace.pid, expr) - return result + return Distributed.remotecall_eval(Main, workspace.pid, expr) catch e return e end diff --git a/src/runner/PlutoRunner.jl b/src/runner/PlutoRunner.jl index 592cb4a9d8..1b2df028d6 100644 --- a/src/runner/PlutoRunner.jl +++ b/src/runner/PlutoRunner.jl @@ -182,9 +182,10 @@ function try_macroexpand(mod, cell_uuid, expr) expanded_expr = macroexpand(mod, expr) elapsed_ns = time_ns() - elapsed_ns - ExpandedCallCells[cell_uuid] = (;expr=no_workspace_ref(expanded_expr), runtime=elapsed_ns) + expr_to_save = no_workspace_ref(expanded_expr) + ExpandedCallCells[cell_uuid] = (;expr=expr_to_save, runtime=elapsed_ns) - return sanitize_expr(expanded_expr) + return (sanitize_expr(expanded_expr), expr_hash(expr_to_save)) catch e return e end @@ -343,14 +344,12 @@ If the third argument is a `Tuple{Set{Symbol}, Set{Symbol}}` containing the refe This function is memoized: running the same expression a second time will simply call the same generated function again. This is much faster than evaluating the expression, because the function only needs to be Julia-compiled once. See https://github.com/fonsp/Pluto.jl/pull/720 """ -function run_expression(m::Module, expr::Any, cell_id::UUID, function_wrapped_info::Union{Nothing,Tuple{Set{Symbol},Set{Symbol}}}=nothing, contains_user_defined_macrocalls::Bool=false) +function run_expression(m::Module, expr::Any, cell_id::UUID, function_wrapped_info::Union{Nothing,Tuple{Set{Symbol},Set{Symbol}}}=nothing, forced_expr_id::Union{ObjectID,Nothing}=nothing, contains_user_defined_macrocalls::Bool=false) currently_running_cell_id[] = cell_id cell_published_objects[cell_id] = Dict{String,Any}() + (expr, expansion_runtime) = pop!(ExpandedCallCells, cell_id, (;expr,runtime=zero(UInt64))) result, runtime = if function_wrapped_info === nothing - expanded = pop!(ExpandedCallCells, cell_id, (;expr,runtime=zero(UInt64))) - expr = expanded.expr - expansion_runtime = expanded.runtime proof = ReturnProof() @@ -372,7 +371,7 @@ function run_expression(m::Module, expr::Any, cell_id::UUID, function_wrapped_in (ans, add_runtimes(runtime, expansion_runtime)) end else - key = expr_hash(expr) + key = forced_expr_id !== nothing ? forced_expr_id : expr_hash(expr) local computer = get(computers, key, nothing) if computer === nothing try @@ -389,7 +388,7 @@ function run_expression(m::Module, expr::Any, cell_id::UUID, function_wrapped_in if (ans isa CapturedException) && (ans.ex isa UndefVarError) run_expression(m, expr, cell_id, nothing) else - ans, runtime + ans, add_runtimes(runtime, expansion_runtime) end end diff --git a/test/MacroAnalysis.jl b/test/MacroAnalysis.jl index 1e90b4a947..a9248f77ce 100644 --- a/test/MacroAnalysis.jl +++ b/test/MacroAnalysis.jl @@ -300,9 +300,7 @@ import Pluto: PlutoRunner, Notebook, WorkspaceManager, Cell, ServerSession, Clie end""") update_run!(🍭, notebook, cell(2)) - # Cell 4 is not re-executed because the ReactiveNode - # for @b(x) has not changed. - @test_broken cell(4).output.body == "42" + @test cell(4).output.body == "42" @test cell(3).errored == true notebook = Notebook(Cell.([ @@ -362,11 +360,9 @@ import Pluto: PlutoRunner, Notebook, WorkspaceManager, Cell, ServerSession, Clie @test cell(1) |> noerror @test cell(2) |> noerror @test cell(3) |> noerror + @test cell(4).output.body != "42" @test cell(4).errored == true - - # cell 5 should re-run because all cells calling @b should be invalidated - # at the reactive node level this is not yet the case - @test_broken noerror(cell(5); verbose=false) + @test cell(5) |> noerror end @testset "Explicitely running macrocalls updates the reactive node" begin @@ -424,17 +420,18 @@ import Pluto: PlutoRunner, Notebook, WorkspaceManager, Cell, ServerSession, Clie @test cell(2) |> noerror @test cell(3) |> noerror @test cell(4) |> noerror + output_1 = cell(4).output.body @test cell(5).errored == true @test cell(6) |> noerror setcode(cell(6), "updater = 2") update_run!(🍭, notebook, cell(6)) - @test cell(4).errored == true - - # Since the run of `@b()` was not explicit, - # the reactive node of cell(1) was not updated :'( - @test_broken noerror(cell(5); verbose=false) + # the output of cell 4 has not changed since the underlying computer + # has not been regenerated. To update the reactive node and macrocall + # an explicit run of @b() must be done. + @test cell(4).output.body == output_1 + @test cell(5).errored == true end @testset "Weird behavior" begin @@ -503,6 +500,52 @@ import Pluto: PlutoRunner, Notebook, WorkspaceManager, Cell, ServerSession, Clie @test cell(1).output.body == "42" end + @testset "Macro with long compile time gets function wrapped" begin + ms = 1e-3 + ns = 1e-9 + sleep_time = 40ms + + notebook = Notebook(Cell.([ + "updater; @b()", + """macro b() + x = rand() + sleep($sleep_time) + :(1+\$x) + end""", + "updater = :slow", + ])) + cell(idx) = notebook.cells[idx] + update_run!(🍭, notebook, notebook.cells) + + @test noerror(cell(1)) + runtime = cell(1).runtime*ns + output_1 = cell(1).output.body + @test sleep_time <= runtime <= 2sleep_time + + setcode(cell(3), "updater = :fast") + update_run!(🍭, notebook, cell(3)) + + @test noerror(cell(1)) + runtime = cell(1).runtime*ns + @test runtime < sleep_time # no recompilation! + + # output is the same because no new compilation happens + @test output_1 == cell(1).output.body + + # force recompilation by explicitely running the cell + update_run!(🍭, notebook, cell(1)) + + @test cell(1) |> noerror + @test output_1 != cell(1).output.body + output_3 = cell(1).output.body + + setcode(cell(1), "@b()") # changing code generates a new πŸ’» + update_run!(🍭, notebook, cell(1)) + + @test cell(1) |> noerror + @test output_3 != cell(1).output.body + end + @testset "Macro Prefix" begin 🍭.options.evaluation.workspace_use_distributed = true From f31a67e9228cb0b1f1b5d7d78ec2cd3f7d6ab4c4 Mon Sep 17 00:00:00 2001 From: Paul Date: Tue, 26 Oct 2021 16:26:05 +0200 Subject: [PATCH 10/28] update comment in `can_be_function_wrapped` --- src/analysis/ExpressionExplorer.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/analysis/ExpressionExplorer.jl b/src/analysis/ExpressionExplorer.jl index d634ab395a..d28dff0593 100644 --- a/src/analysis/ExpressionExplorer.jl +++ b/src/analysis/ExpressionExplorer.jl @@ -1226,7 +1226,9 @@ function can_be_function_wrapped(x::Expr) x.head === :module || x.head === :function || x.head === :macro || - x.head === :macrocall || # we might want to get rid of this one, but that requires some work + # Cells containing macrocalls will actually be function wrapped using the expanded version of the expression + # See https://github.com/fonsp/Pluto.jl/pull/1597 + x.head === :macrocall || x.head === :struct || x.head === :abstract || (x.head === :(=) && is_function_assignment(x)) || # f(x) = ... From a5bbe95f53b913788f11cc8c614625f81a32b2f5 Mon Sep 17 00:00:00 2001 From: Paul Date: Tue, 26 Oct 2021 16:46:04 +0200 Subject: [PATCH 11/28] remove upper bound in timing test --- test/MacroAnalysis.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/MacroAnalysis.jl b/test/MacroAnalysis.jl index a9248f77ce..58d5c9b102 100644 --- a/test/MacroAnalysis.jl +++ b/test/MacroAnalysis.jl @@ -520,7 +520,7 @@ import Pluto: PlutoRunner, Notebook, WorkspaceManager, Cell, ServerSession, Clie @test noerror(cell(1)) runtime = cell(1).runtime*ns output_1 = cell(1).output.body - @test sleep_time <= runtime <= 2sleep_time + @test sleep_time <= runtime setcode(cell(3), "updater = :fast") update_run!(🍭, notebook, cell(3)) From e77142ab92e66de304687808ba8fadf8780a7877 Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 28 Oct 2021 14:32:47 +0200 Subject: [PATCH 12/28] use cell UUIDs as computer keys --- src/runner/PlutoRunner.jl | 41 +++++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/src/runner/PlutoRunner.jl b/src/runner/PlutoRunner.jl index 2ca37b3070..bd30e90e9f 100644 --- a/src/runner/PlutoRunner.jl +++ b/src/runner/PlutoRunner.jl @@ -224,6 +224,8 @@ const return_error = "Pluto: You can only use return inside a function." struct Computer f::Function + expr_id::ObjectID + cleanup_funcs::Set{Function} return_proof::ReturnProof input_globals::Vector{Symbol} output_globals::Vector{Symbol} @@ -231,13 +233,13 @@ end expr_hash(e::Expr) = objectid(e.head) + mapreduce(p -> objectid((p[1], expr_hash(p[2]))), +, enumerate(e.args); init=zero(ObjectID)) expr_hash(x) = objectid(x) -# TODO: clear key when a cell is deleted furever -const computers = Dict{ObjectID,Computer}() +const computers = Dict{UUID,Computer}() const computer_workspace = Main -function register_computer(expr::Expr, key, input_globals::Vector{Symbol}, output_globals::Vector{Symbol}) +"Registers a new computer for the cell, cleaning up the old one if there is one." +function register_computer(expr::Expr, key::ObjectID, cell_id::UUID, input_globals::Vector{Symbol}, output_globals::Vector{Symbol}) proof = ReturnProof() @gensym result @@ -251,7 +253,29 @@ function register_computer(expr::Expr, key, input_globals::Vector{Symbol}, outpu f = Core.eval(computer_workspace, e) - computers[key] = Computer(f, proof, input_globals, output_globals) + if haskey(computers, cell_id) + delete_computer!(computers, cell_id) + end + + computers[cell_id] = Computer(f, key, Set{Function}([]), proof, input_globals, output_globals) +end + +function delete_computer!(computers::Dict{UUID,Computer}, cell_id::UUID) + computer = pop!(computers, cell_id) + for cleanup_func in computer.cleanup_funcs + cleanup_func() + end + Base.visit(Base.delete_method, methods(computer.f).mt) # Make the computer function uncallable +end + +parse_cell_id(filename::Symbol) = filename |> string |> parse_cell_id +parse_cell_id(filename::AbstractString) = + match(r"#==#(.*)", filename).captures |> only |> UUID + +function register_cleanup(f::Function, cell_id::UUID) + @assert haskey(computers, cell_id) "The cell $cell_id is not function wrapped" + push!(computers[cell_id].cleanup_funcs, f) + nothing end quote_if_needed(x) = x @@ -371,11 +395,11 @@ function run_expression(m::Module, expr::Any, cell_id::UUID, function_wrapped_in (ans, add_runtimes(runtime, expansion_runtime)) end else - key = forced_expr_id !== nothing ? forced_expr_id : expr_hash(expr) - local computer = get(computers, key, nothing) - if computer === nothing + expr_id = forced_expr_id !== nothing ? forced_expr_id : expr_hash(expr) + local computer = get(computers, cell_id, nothing) + if computer === nothing || computer.expr_id !== expr_id try - computer = register_computer(expr, key, collect.(function_wrapped_info)...) + computer = register_computer(expr, expr_id, cell_id, collect.(function_wrapped_info)...) catch e # @error "Failed to generate computer function" expr exception=(e,stacktrace(catch_backtrace())) return run_expression(m, expr, cell_id, nothing) @@ -386,6 +410,7 @@ function run_expression(m::Module, expr::Any, cell_id::UUID, function_wrapped_in # This check solves the problem of a cell like `false && variable_that_does_not_exist`. This should run without error, but will fail in our function-wrapping-magic because we get the value of `variable_that_does_not_exist` before calling the generated function. # The fix is to detect this situation and run the expression in the classical way. if (ans isa CapturedException) && (ans.ex isa UndefVarError) + # @warn "Got variable that does not exist" ex=ans run_expression(m, expr, cell_id, nothing) else ans, add_runtimes(runtime, expansion_runtime) From 18b3517c5567410c0f35fbd2d88e4e79c7d2c56b Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 28 Oct 2021 21:28:54 +0200 Subject: [PATCH 13/28] self updating channel --- src/evaluation/WorkspaceManager.jl | 21 +++++++++++++++++++++ src/runner/PlutoRunner.jl | 25 ++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/src/evaluation/WorkspaceManager.jl b/src/evaluation/WorkspaceManager.jl index b513f4ebff..921b8fa266 100644 --- a/src/evaluation/WorkspaceManager.jl +++ b/src/evaluation/WorkspaceManager.jl @@ -1,5 +1,6 @@ module WorkspaceManager import UUIDs: UUID +import ..Pluto import ..Pluto: Configuration, Notebook, Cell, ProcessStatus, ServerSession, ExpressionExplorer, pluto_filename, Token, withtoken, Promise, tamepath, project_relative_path, putnotebookupdates!, UpdateMessage import ..Pluto.PkgCompat import ..Configuration: CompilerOptions, _merge_notebook_compiler_options, _convert_to_flags @@ -58,6 +59,9 @@ function make_workspace((session, notebook)::SN; force_offline::Bool=false)::Wor log_channel = Core.eval(Main, quote $(Distributed).RemoteChannel(() -> eval(:(Main.PlutoRunner.log_channel)), $pid) end) + run_channel = Core.eval(Main, quote + $(Distributed).RemoteChannel(() -> eval(:(Main.PlutoRunner.run_channel)), $pid) + end) module_name = create_emptyworkspacemodule(pid) original_LOAD_PATH, original_ACTIVE_PROJECT = Distributed.remotecall_eval(Main, pid, :(Base.LOAD_PATH, Base.ACTIVE_PROJECT[])) @@ -71,6 +75,7 @@ function make_workspace((session, notebook)::SN; force_offline::Bool=false)::Wor ) @async start_relaying_logs((session, notebook), log_channel) + @async start_relaying_self_updates((session, notebook), run_channel) cd_workspace(workspace, notebook.path) use_nbpkg_environment((session, notebook), workspace) @@ -103,6 +108,22 @@ function use_nbpkg_environment((session, notebook)::SN, workspace=nothing) end end +function start_relaying_self_updates((session, notebook)::SN, run_channel::Distributed.RemoteChannel) + while true + try + next_run_uuid = take!(run_channel) + + cell_to_run = notebook.cells_dict[next_run_uuid] + Pluto.run_reactive!(session, notebook, notebook.topology, notebook.topology, Cell[cell_to_run]; persist_js_state=true) + catch e + if !isopen(run_channel) + break + end + @error "Failed to relay self-update" exception=(e, catch_backtrace()) + end + end +end + function start_relaying_logs((session, notebook)::SN, log_channel::Distributed.RemoteChannel) while true try diff --git a/src/runner/PlutoRunner.jl b/src/runner/PlutoRunner.jl index bd30e90e9f..38738d0df2 100644 --- a/src/runner/PlutoRunner.jl +++ b/src/runner/PlutoRunner.jl @@ -237,7 +237,6 @@ expr_hash(x) = objectid(x) const computers = Dict{UUID,Computer}() const computer_workspace = Main - "Registers a new computer for the cell, cleaning up the old one if there is one." function register_computer(expr::Expr, key::ObjectID, cell_id::UUID, input_globals::Vector{Symbol}, output_globals::Vector{Symbol}) proof = ReturnProof() @@ -424,7 +423,31 @@ function run_expression(m::Module, expr::Any, cell_id::UUID, function_wrapped_in cell_results[cell_id], cell_runtimes[cell_id] = result, runtime end +# Channel to trigger implicits run +const run_channel = Channel{UUID}(10) + +# internal api, be careful as this can trigger an infinite loop +function _self_run(cell_id::UUID) + # if cell_id != currently_running_cell_id[] + # @warn "_self_run($cell_id) called from outside the cell (from $(currently_running_cell_id[])), this can lead to infinite loops" + # end + # make sure only one of this cell_id is in the run channel + # by emptying it and filling it again + new_uuids = UUID[] + while isready(run_channel) + uuid = take!(run_channel) + if uuid != cell_id + push!(new_uuids, uuid) + end + end + size = length(new_uuids) + for uuid in new_uuids + put!(run_channel, uuid) + end + + put!(run_channel, cell_id) +end From 43b9092d16df6ea139a2457ba5086c0fce352871 Mon Sep 17 00:00:00 2001 From: Michiel Dral Date: Fri, 29 Oct 2021 17:41:13 +0000 Subject: [PATCH 14/28] Macros independent of computers! --- src/analysis/DependencyCache.jl | 9 -- src/evaluation/Run.jl | 47 +++++++---- src/evaluation/WorkspaceManager.jl | 8 +- src/notebook/Cell.jl | 3 +- src/runner/PlutoRunner.jl | 127 +++++++++++++++++++---------- 5 files changed, 117 insertions(+), 77 deletions(-) diff --git a/src/analysis/DependencyCache.jl b/src/analysis/DependencyCache.jl index b73d630d97..00ee819a09 100644 --- a/src/analysis/DependencyCache.jl +++ b/src/analysis/DependencyCache.jl @@ -28,21 +28,12 @@ function upstream_cells_map(cell::Cell, notebook::Notebook)::Dict{Symbol,Vector{ ) end -"Checks whether or not the cell references user-defined macrocalls" -function contains_user_defined_macrocalls(cell::Cell, notebook::Notebook)::Bool - calls = notebook.topology.nodes[cell].macrocalls - !isempty(calls) && any(notebook.cells) do other - !disjoint(notebook.topology.nodes[other].funcdefs_without_signatures, calls) - end -end - "Fills cell dependency information for display in the GUI" function update_dependency_cache!(cell::Cell, notebook::Notebook) cell.cell_dependencies = CellDependencies( downstream_cells_map(cell, notebook), upstream_cells_map(cell, notebook), cell_precedence_heuristic(notebook.topology, cell), - contains_user_defined_macrocalls(cell, notebook) ) end diff --git a/src/evaluation/Run.jl b/src/evaluation/Run.jl index 7b1c53f0d4..edbaa85561 100644 --- a/src/evaluation/Run.jl +++ b/src/evaluation/Run.jl @@ -201,7 +201,6 @@ function run_single!(session_notebook::Union{Tuple{ServerSession,Notebook},Works ends_with_semicolon(cell.code), expr_cache.function_wrapped ? (filter(!is_joined_funcname, reactive_node.references), reactive_node.definitions) : nothing, expr_cache.forced_expr_id, - cell.cell_dependencies.contains_user_defined_macrocalls, ) set_output!(cell, run, expr_cache; persist_js_state=persist_js_state) if session_notebook isa Tuple && run.process_exited @@ -248,6 +247,15 @@ function can_help_resolve_cells(topology::NotebookTopology, cell::Cell) any(is_macro_identifier, cell_node.funcdefs_without_signatures) end +# Sorry couldn't help myself - DRAL +abstract type Result end +struct Success <: Result + result +end +struct Failure <: Result + error +end + """We still have 'unresolved' macrocalls, use the current and maybe previous workspace to do macro-expansions. You can optionally specify the roots for the current reactive run. If a cell macro contains only macros that will @@ -267,31 +275,37 @@ function resolve_topology( sn = (session, notebook) function macroexpand_cell(cell) - try_macroexpand(module_name::Union{Nothing,Symbol}=nothing) = - macroexpand_in_workspace(sn, unresolved_topology.codes[cell].parsedcode, cell.cell_id, module_name) + try_macroexpand(module_name::Union{Nothing,Symbol}=nothing) = try + Success(macroexpand_in_workspace(sn, unresolved_topology.codes[cell].parsedcode, cell.cell_id, module_name)) + catch e + Failure(e) + end - res = try_macroexpand() - if (res isa LoadError && res.error isa UndefVarError) || res isa UndefVarError - res = try_macroexpand(old_workspace_name) + result = try_macroexpand() + if result isa Success + result + else + if (result.error isa LoadError && result.error.error isa UndefVarError) || result.error isa UndefVarError + try_macroexpand(old_workspace_name) + else + result + end end - res end - # nothing means that the expansion has failed function analyze_macrocell(cell::Cell) if unresolved_topology.nodes[cell].macrocalls βŠ† ExpressionExplorer.can_macroexpand return nothing end result = macroexpand_cell(cell) - if result isa Exception - @debug "Expansion failed" err=result - nothing - else # otherwise, we use the expanded expression + the list of macrocalls - (expr, computer_id) = result + if result isa Success + (expr, computer_id) = result.result expanded_node = ExpressionExplorer.try_compute_symbolreferences(expr) |> ReactiveNode function_wrapped = ExpressionExplorer.can_be_function_wrapped(expr) - (expanded_node, function_wrapped, computer_id) + Success((expanded_node, function_wrapped, computer_id)) + else + result end end @@ -310,8 +324,8 @@ function resolve_topology( end result = analyze_macrocell(cell) - if result !== nothing - (new_node, function_wrapped, forced_expr_id) = result + if result isa Success + (new_node, function_wrapped, forced_expr_id) = result.result union!(new_node.macrocalls, unresolved_topology.nodes[cell].macrocalls) union!(new_node.references, new_node.macrocalls) new_nodes[cell] = new_node @@ -319,6 +333,7 @@ function resolve_topology( # set function_wrapped to the function wrapped analysis of the expanded expression. new_codes[cell] = ExprAnalysisCache(unresolved_topology.codes[cell]; forced_expr_id, function_wrapped) else + @debug "Expansion failed" err=result.error push!(still_unresolved_nodes, cell) end end diff --git a/src/evaluation/WorkspaceManager.jl b/src/evaluation/WorkspaceManager.jl index 921b8fa266..77e6f6fee5 100644 --- a/src/evaluation/WorkspaceManager.jl +++ b/src/evaluation/WorkspaceManager.jl @@ -277,7 +277,6 @@ function eval_format_fetch_in_workspace( ends_with_semicolon::Bool=false, function_wrapped_info::Union{Nothing,Tuple}=nothing, forced_expr_id::Union{PlutoRunner.ObjectID,Nothing}=nothing, - contains_user_defined_macrocalls::Bool=false )::NamedTuple{(:output_formatted, :errored, :interrupted, :process_exited, :runtime, :published_objects),Tuple{PlutoRunner.MimedOutput,Bool,Bool,Bool,Union{UInt64,Nothing},Dict{String,Any}}} workspace = get_workspace(session_notebook) @@ -302,7 +301,6 @@ function eval_format_fetch_in_workspace( $cell_id, $function_wrapped_info, $forced_expr_id, - $contains_user_defined_macrocalls, ))) put!(workspace.dowork_token) nothing @@ -360,11 +358,7 @@ function macroexpand_in_workspace(session_notebook::Union{SN,Workspace}, macroca expr = quote PlutoRunner.try_macroexpand($(module_name), $(cell_uuid), $(macrocall |> QuoteNode)) end - try - return Distributed.remotecall_eval(Main, workspace.pid, expr) - catch e - return e - end + return Distributed.remotecall_eval(Main, workspace.pid, expr) end "Evaluate expression inside the workspace - output is returned. For internal use." diff --git a/src/notebook/Cell.jl b/src/notebook/Cell.jl index 6e881342fe..6edabccc78 100644 --- a/src/notebook/Cell.jl +++ b/src/notebook/Cell.jl @@ -17,7 +17,6 @@ struct CellDependencies{T} # T == Cell, but this has to be parametric to avoid a downstream_cells_map::Dict{Symbol,Vector{T}} upstream_cells_map::Dict{Symbol,Vector{T}} precedence_heuristic::Int - contains_user_defined_macrocalls::Bool end "The building block of a `Notebook`. Contains code, output, reactivity data, mitochondria and ribosomes." @@ -38,7 +37,7 @@ Base.@kwdef mutable struct Cell runtime::Union{Nothing,UInt64}=nothing # note that this field might be moved somewhere else later. If you are interested in visualizing the cell dependencies, take a look at the cell_dependencies field in the frontend instead. - cell_dependencies::CellDependencies{Cell}=CellDependencies{Cell}(Dict{Symbol,Vector{Cell}}(), Dict{Symbol,Vector{Cell}}(), 99, false) + cell_dependencies::CellDependencies{Cell}=CellDependencies{Cell}(Dict{Symbol,Vector{Cell}}(), Dict{Symbol,Vector{Cell}}(), 99) running_disabled::Bool=false depends_on_disabled_cells::Bool=false diff --git a/src/runner/PlutoRunner.jl b/src/runner/PlutoRunner.jl index 38738d0df2..3c72a238d4 100644 --- a/src/runner/PlutoRunner.jl +++ b/src/runner/PlutoRunner.jl @@ -26,7 +26,14 @@ export @bind MimedOutput = Tuple{Union{String,Vector{UInt8},Dict{Symbol,Any}},MIME} const ObjectID = typeof(objectid("hello computer")) const ObjectDimPair = Tuple{ObjectID,Int64} -const ExpandedCallCells = Dict{UUID,NamedTuple{(:expr,:runtime),Tuple{Expr,UInt64}}}() + +Base.@kwdef struct CachedMacroExpansion + original_expr::Expr + expanded_expr::Expr + expansion_duration::UInt64 + did_mention_expansion_time::Bool=false +end +const cell_expanded_exprs = Dict{UUID,CachedMacroExpansion}() @@ -87,10 +94,15 @@ function wrap_dot(name) end """ -Returns an Expr with no GlobalRef to `Main.workspaceXX` so that reactive updates will work. + no_workspace_ref(ref::Union{GlobalRef,Expr}, mod_name::Symbol=nothing) + +Goes through an expression and removes all "global" references to workspace modules (e.g. Main.workspace#XXX). + +This is useful for us because when we macroexpand, the global refs will normally point to the module it was built in. +We don't re-build the macro in every workspace, so we need to remove these refs manually in order to point to the new module instead. + +TODO? Don't remove the refs, but instead replace them with a new ref pointing to the new module? """ -no_workspace_ref(other, _=nothing) = other -no_workspace_ref(expr::Expr, mod_name=nothing) = Expr(expr.head, map(arg -> no_workspace_ref(arg, mod_name), expr.args)...) function no_workspace_ref(ref::GlobalRef, mod_name=nothing) test_mod_name = nameof(ref.mod) |> string if startswith(test_mod_name, "workspace#") && @@ -102,6 +114,8 @@ function no_workspace_ref(ref::GlobalRef, mod_name=nothing) ref end end +no_workspace_ref(expr::Expr, mod_name=nothing) = Expr(expr.head, map(arg -> no_workspace_ref(arg, mod_name), expr.args)...) +no_workspace_ref(other, _=nothing) = other function sanitize_expr(symbol::Symbol) symbol @@ -177,18 +191,21 @@ sanitize_value(other) = sanitize_expr(other) function try_macroexpand(mod, cell_uuid, expr) - try - elapsed_ns = time_ns() - expanded_expr = macroexpand(mod, expr) - elapsed_ns = time_ns() - elapsed_ns - - expr_to_save = no_workspace_ref(expanded_expr) - ExpandedCallCells[cell_uuid] = (;expr=expr_to_save, runtime=elapsed_ns) + elapsed_ns = time_ns() + expanded_expr = macroexpand(mod, expr) + elapsed_ns = time_ns() - elapsed_ns + + # Removes baked in references to the module this was macroexpanded in. + # Fix for https://github.com/fonsp/Pluto.jl/issues/1112 + expr_to_save = no_workspace_ref(expanded_expr) + + cell_expanded_exprs[cell_uuid] = CachedMacroExpansion( + original_expr=expr, + expanded_expr=expr_to_save, + expansion_duration=elapsed_ns + ) - return (sanitize_expr(expanded_expr), expr_hash(expr_to_save)) - catch e - return e - end + return (sanitize_expr(expanded_expr), expr_hash(expr_to_save)) end function get_module_names(workspace_module, module_ex::Expr) @@ -350,16 +367,17 @@ function run_inside_trycatch(m::Module, f::Union{Expr,Function}, return_proof::R end end -function timed_visit_expand(current_module::Module, expr::Expr)::Tuple{Expr,UInt64} - elapsed = time_ns() - expanded_expr = no_workspace_ref(macroexpand(current_module, expr), nameof(current_module)) # visit_expand(current_module, expr) - elapsed = time_ns() - elapsed - (expanded_expr, elapsed) -end - add_runtimes(::Nothing, ::UInt64) = nothing add_runtimes(a::UInt64, b::UInt64) = a+b +contains_macrocall(expr::Expr) = if expr.head === :macrocall + true +else + any(arg -> contains_macrocall(arg), expr.args) +end +contains_macrocall(other) = false + + """ Run the given expression in the current workspace module. If the third argument is `nothing`, then the expression will be `Core.eval`ed. The result and runtime are stored inside [`cell_results`](@ref) and [`cell_runtimes`](@ref). @@ -367,32 +385,53 @@ If the third argument is a `Tuple{Set{Symbol}, Set{Symbol}}` containing the refe This function is memoized: running the same expression a second time will simply call the same generated function again. This is much faster than evaluating the expression, because the function only needs to be Julia-compiled once. See https://github.com/fonsp/Pluto.jl/pull/720 """ -function run_expression(m::Module, expr::Any, cell_id::UUID, function_wrapped_info::Union{Nothing,Tuple{Set{Symbol},Set{Symbol}}}=nothing, forced_expr_id::Union{ObjectID,Nothing}=nothing, contains_user_defined_macrocalls::Bool=false) +function run_expression(m::Module, expr::Any, cell_id::UUID, function_wrapped_info::Union{Nothing,Tuple{Set{Symbol},Set{Symbol}}}=nothing, forced_expr_id::Union{ObjectID,Nothing}=nothing) currently_running_cell_id[] = cell_id cell_published_objects[cell_id] = Dict{String,Any}() - (expr, expansion_runtime) = pop!(ExpandedCallCells, cell_id, (;expr,runtime=zero(UInt64))) - result, runtime = if function_wrapped_info === nothing + # If the cell contains macro calls, we want those macro calls to preserve their identity, + # so we macroexpand this earlier (during expression explorer stuff), and then we find it here. + # NOTE Turns out sometimes there is no macroexpanded version even though the expression contains macro calls... + # .... So I macroexpand when there is no cached version just to be sure πŸ€·β€β™€οΈ + original_expr = expr + if !haskey(cell_expanded_exprs, cell_id) || cell_expanded_exprs[cell_id].original_expr != original_expr + try + try_macroexpand(m, cell_id, expr) + catch e + bt = stacktrace(catch_backtrace()) + result = CapturedException(e, bt) + cell_results[cell_id], cell_runtimes[cell_id] = (result, nothing) + return (result, nothing) + end + end + expanded_cache = cell_expanded_exprs[cell_id] + expr = expanded_cache.expanded_expr + # Only mention the expansion time once + expansion_runtime = if expanded_cache.did_mention_expansion_time === false + # Is this really the easiest way to clone a struct with some changes? Pfffft + cell_expanded_exprs[cell_id] = CachedMacroExpansion( + original_expr=expanded_cache.original_expr, + expanded_expr=expanded_cache.expanded_expr, + expansion_duration=expanded_cache.expansion_duration, + did_mention_expansion_time=true, + ) + expanded_cache.expansion_duration + else + zero(UInt64) + end + + if contains_macrocall(expr) + @error "Expression contains a macrocall" expr + throw("Expression still contains macro calls!!") + end + + result, runtime = if function_wrapped_info === nothing proof = ReturnProof() - # Note: fix for https://github.com/fonsp/Pluto.jl/issues/1112 - if contains_user_defined_macrocalls - try - (expr, other_expansion_runtime) = timed_visit_expand(m, expr) - expansion_runtime = max(expansion_runtime, other_expansion_runtime) - wrapped = timed_expr(expr, proof) - ans, runtime = run_inside_trycatch(m, wrapped, proof) - (ans, add_runtimes(runtime, expansion_runtime)) - catch ex - bt = stacktrace(catch_backtrace()) - (CapturedException(ex, bt), nothing) - end - else - wrapped = timed_expr(expr, proof) - ans, runtime = run_inside_trycatch(m, wrapped, proof) - (ans, add_runtimes(runtime, expansion_runtime)) - end + wrapped = timed_expr(expr, proof) + ans, runtime = run_inside_trycatch(m, wrapped, proof) + (ans, add_runtimes(runtime, expansion_runtime)) else expr_id = forced_expr_id !== nothing ? forced_expr_id : expr_hash(expr) local computer = get(computers, cell_id, nothing) @@ -407,7 +446,9 @@ function run_expression(m::Module, expr::Any, cell_id::UUID, function_wrapped_in ans, runtime = run_inside_trycatch(m, () -> compute(m, computer), computer.return_proof) # This check solves the problem of a cell like `false && variable_that_does_not_exist`. This should run without error, but will fail in our function-wrapping-magic because we get the value of `variable_that_does_not_exist` before calling the generated function. - # The fix is to detect this situation and run the expression in the classical way. + # The fix is to "detect" this situation and run the expression in the classical way. + # TODO This error originates from our own `getfield.([m], input_globals)`... we can put a check there that + # .... throws a more specific error instead of this one that could come from inside the cell code as well. if (ans isa CapturedException) && (ans.ex isa UndefVarError) # @warn "Got variable that does not exist" ex=ans run_expression(m, expr, cell_id, nothing) From 3348601216593fe8fb17586626ae86044b71a0e2 Mon Sep 17 00:00:00 2001 From: Michiel Dral Date: Fri, 29 Oct 2021 17:58:43 +0000 Subject: [PATCH 15/28] Make the "Undefined variable" check in computers a bit less hacky --- src/runner/PlutoRunner.jl | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/runner/PlutoRunner.jl b/src/runner/PlutoRunner.jl index 3c72a238d4..9d5d7a358f 100644 --- a/src/runner/PlutoRunner.jl +++ b/src/runner/PlutoRunner.jl @@ -393,8 +393,7 @@ function run_expression(m::Module, expr::Any, cell_id::UUID, function_wrapped_in # so we macroexpand this earlier (during expression explorer stuff), and then we find it here. # NOTE Turns out sometimes there is no macroexpanded version even though the expression contains macro calls... # .... So I macroexpand when there is no cached version just to be sure πŸ€·β€β™€οΈ - original_expr = expr - if !haskey(cell_expanded_exprs, cell_id) || cell_expanded_exprs[cell_id].original_expr != original_expr + if !haskey(cell_expanded_exprs, cell_id) || cell_expanded_exprs[cell_id].original_expr != expr try try_macroexpand(m, cell_id, expr) catch e @@ -405,9 +404,12 @@ function run_expression(m::Module, expr::Any, cell_id::UUID, function_wrapped_in end end + # We can be sure there is a cached expression now, yay expanded_cache = cell_expanded_exprs[cell_id] expr = expanded_cache.expanded_expr - # Only mention the expansion time once + + # We add the time it took to macroexpand to the time for the first call, + # but we make sure we don't mention it on subsequent calls expansion_runtime = if expanded_cache.did_mention_expansion_time === false # Is this really the easiest way to clone a struct with some changes? Pfffft cell_expanded_exprs[cell_id] = CachedMacroExpansion( @@ -443,18 +445,18 @@ function run_expression(m::Module, expr::Any, cell_id::UUID, function_wrapped_in return run_expression(m, expr, cell_id, nothing) end end - ans, runtime = run_inside_trycatch(m, () -> compute(m, computer), computer.return_proof) # This check solves the problem of a cell like `false && variable_that_does_not_exist`. This should run without error, but will fail in our function-wrapping-magic because we get the value of `variable_that_does_not_exist` before calling the generated function. - # The fix is to "detect" this situation and run the expression in the classical way. - # TODO This error originates from our own `getfield.([m], input_globals)`... we can put a check there that - # .... throws a more specific error instead of this one that could come from inside the cell code as well. - if (ans isa CapturedException) && (ans.ex isa UndefVarError) - # @warn "Got variable that does not exist" ex=ans + # The fix is to detect this situation and run the expression in the classical way. + ans, runtime = if any(name -> !isdefined(m, name), computer.input_globals) + # Do run_expression but with function_wrapped_info=nothing so it doesn't go in a Computer() + # @warn "Got variables that don't exist, running outside of computer" not_existing=filter(name -> !isdefined(m, name), computer.input_globals) run_expression(m, expr, cell_id, nothing) else - ans, add_runtimes(runtime, expansion_runtime) + run_inside_trycatch(m, () -> compute(m, computer), computer.return_proof) end + + ans, add_runtimes(runtime, expansion_runtime) end if (result isa CapturedException) && (result.ex isa InterruptException) From 282b839ed6d6d46aa02f1aecaf37b1d5056353e7 Mon Sep 17 00:00:00 2001 From: Michiel Dral Date: Fri, 29 Oct 2021 18:07:12 +0000 Subject: [PATCH 16/28] Make Computers work with `if false; x = 10; end` --- src/runner/PlutoRunner.jl | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/runner/PlutoRunner.jl b/src/runner/PlutoRunner.jl index 9d5d7a358f..1abc096b66 100644 --- a/src/runner/PlutoRunner.jl +++ b/src/runner/PlutoRunner.jl @@ -263,7 +263,7 @@ function register_computer(expr::Expr, key::ObjectID, cell_id::UUID, input_globa Expr(:(=), result, timed_expr(expr, proof)), Expr(:tuple, result, - Expr(:tuple, output_globals...) + Expr(:tuple, map(x -> :(@isdefined($(x)) ? $(x) : $(NotDefined())),output_globals)...) ) )) @@ -297,6 +297,8 @@ end quote_if_needed(x) = x quote_if_needed(x::Union{Expr, Symbol, QuoteNode, LineNumberNode}) = QuoteNode(x) +struct NotDefined end + function compute(m::Module, computer::Computer) # 1. get the referenced global variables # this might error if the global does not exist, which is exactly what we want @@ -308,7 +310,12 @@ function compute(m::Module, computer::Computer) result, output_global_values = out for (name, val) in zip(computer.output_globals, output_global_values) - Core.eval(m, Expr(:(=), name, quote_if_needed(val))) + # Core.eval(m, Expr(:(=), name, quote_if_needed(val))) + Core.eval(m, quote + if $(quote_if_needed(val)) === $(NotDefined()) + $(name) = $(quote_if_needed(val)) + end + end) end result From c8e007d7a789c4cb10bdeec6fcbdd349f5f5a1d4 Mon Sep 17 00:00:00 2001 From: Michiel Dral Date: Fri, 29 Oct 2021 18:10:12 +0000 Subject: [PATCH 17/28] I KEEP DOING THIS --- src/runner/PlutoRunner.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/runner/PlutoRunner.jl b/src/runner/PlutoRunner.jl index 1abc096b66..fdf7582097 100644 --- a/src/runner/PlutoRunner.jl +++ b/src/runner/PlutoRunner.jl @@ -263,7 +263,7 @@ function register_computer(expr::Expr, key::ObjectID, cell_id::UUID, input_globa Expr(:(=), result, timed_expr(expr, proof)), Expr(:tuple, result, - Expr(:tuple, map(x -> :(@isdefined($(x)) ? $(x) : $(NotDefined())),output_globals)...) + Expr(:tuple, map(x -> :(@isdefined($(x)) ? $(x) : $(OutputNotDefined())), output_globals)...) ) )) @@ -297,7 +297,7 @@ end quote_if_needed(x) = x quote_if_needed(x::Union{Expr, Symbol, QuoteNode, LineNumberNode}) = QuoteNode(x) -struct NotDefined end +struct OutputNotDefined end function compute(m::Module, computer::Computer) # 1. get the referenced global variables @@ -312,7 +312,7 @@ function compute(m::Module, computer::Computer) for (name, val) in zip(computer.output_globals, output_global_values) # Core.eval(m, Expr(:(=), name, quote_if_needed(val))) Core.eval(m, quote - if $(quote_if_needed(val)) === $(NotDefined()) + if $(quote_if_needed(val)) !== $(OutputNotDefined()) $(name) = $(quote_if_needed(val)) end end) From 76b3a888a27bf34da542b5e460720d0d2a0b9bbc Mon Sep 17 00:00:00 2001 From: Michiel Dral Date: Fri, 29 Oct 2021 22:20:27 +0000 Subject: [PATCH 18/28] Ah this is what expr_hash is for --- src/runner/PlutoRunner.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/runner/PlutoRunner.jl b/src/runner/PlutoRunner.jl index fdf7582097..ae5f8279a8 100644 --- a/src/runner/PlutoRunner.jl +++ b/src/runner/PlutoRunner.jl @@ -28,7 +28,7 @@ const ObjectID = typeof(objectid("hello computer")) const ObjectDimPair = Tuple{ObjectID,Int64} Base.@kwdef struct CachedMacroExpansion - original_expr::Expr + original_expr_hash::UInt64 expanded_expr::Expr expansion_duration::UInt64 did_mention_expansion_time::Bool=false @@ -200,7 +200,7 @@ function try_macroexpand(mod, cell_uuid, expr) expr_to_save = no_workspace_ref(expanded_expr) cell_expanded_exprs[cell_uuid] = CachedMacroExpansion( - original_expr=expr, + original_expr_hash=expr_hash(expr), expanded_expr=expr_to_save, expansion_duration=elapsed_ns ) @@ -400,7 +400,7 @@ function run_expression(m::Module, expr::Any, cell_id::UUID, function_wrapped_in # so we macroexpand this earlier (during expression explorer stuff), and then we find it here. # NOTE Turns out sometimes there is no macroexpanded version even though the expression contains macro calls... # .... So I macroexpand when there is no cached version just to be sure πŸ€·β€β™€οΈ - if !haskey(cell_expanded_exprs, cell_id) || cell_expanded_exprs[cell_id].original_expr != expr + if !haskey(cell_expanded_exprs, cell_id) || cell_expanded_exprs[cell_id].original_expr_hash != expr_hash(expr) try try_macroexpand(m, cell_id, expr) catch e @@ -420,7 +420,7 @@ function run_expression(m::Module, expr::Any, cell_id::UUID, function_wrapped_in expansion_runtime = if expanded_cache.did_mention_expansion_time === false # Is this really the easiest way to clone a struct with some changes? Pfffft cell_expanded_exprs[cell_id] = CachedMacroExpansion( - original_expr=expanded_cache.original_expr, + original_expr_hash=expanded_cache.original_expr_hash, expanded_expr=expanded_cache.expanded_expr, expansion_duration=expanded_cache.expansion_duration, did_mention_expansion_time=true, From ae2c9f694aadef0c9b0d7b145f051b356e3032c1 Mon Sep 17 00:00:00 2001 From: Michiel Dral Date: Sat, 30 Oct 2021 00:30:00 +0000 Subject: [PATCH 19/28] Maybe maybe maybe --- src/evaluation/Run.jl | 11 +++++++---- src/evaluation/WorkspaceManager.jl | 13 ++++++++----- src/runner/PlutoRunner.jl | 4 ++-- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/evaluation/Run.jl b/src/evaluation/Run.jl index edbaa85561..6aab42990f 100644 --- a/src/evaluation/Run.jl +++ b/src/evaluation/Run.jl @@ -275,10 +275,13 @@ function resolve_topology( sn = (session, notebook) function macroexpand_cell(cell) - try_macroexpand(module_name::Union{Nothing,Symbol}=nothing) = try - Success(macroexpand_in_workspace(sn, unresolved_topology.codes[cell].parsedcode, cell.cell_id, module_name)) - catch e - Failure(e) + try_macroexpand(module_name::Union{Nothing,Symbol}=nothing) = begin + success, result = macroexpand_in_workspace(sn, unresolved_topology.codes[cell].parsedcode, cell.cell_id, module_name) + if success + Success(result) + else + Failure(result) + end end result = try_macroexpand() diff --git a/src/evaluation/WorkspaceManager.jl b/src/evaluation/WorkspaceManager.jl index 77e6f6fee5..b789893cae 100644 --- a/src/evaluation/WorkspaceManager.jl +++ b/src/evaluation/WorkspaceManager.jl @@ -351,14 +351,17 @@ function collect_soft_definitions(session_notebook::SN, modules::Set{Expr}) end -function macroexpand_in_workspace(session_notebook::Union{SN,Workspace}, macrocall, cell_uuid, module_name = nothing) +function macroexpand_in_workspace(session_notebook::Union{SN,Workspace}, macrocall, cell_uuid, module_name = nothing)::Tuple{Bool, Any} workspace = get_workspace(session_notebook) module_name = module_name === nothing ? workspace.module_name : module_name - expr = quote - PlutoRunner.try_macroexpand($(module_name), $(cell_uuid), $(macrocall |> QuoteNode)) - end - return Distributed.remotecall_eval(Main, workspace.pid, expr) + Distributed.remotecall_eval(Main, workspace.pid, quote + try + (true, PlutoRunner.try_macroexpand($(module_name), $(cell_uuid), $(macrocall |> QuoteNode))) + catch e + (false, e) + end + end) end "Evaluate expression inside the workspace - output is returned. For internal use." diff --git a/src/runner/PlutoRunner.jl b/src/runner/PlutoRunner.jl index ae5f8279a8..763d51a40b 100644 --- a/src/runner/PlutoRunner.jl +++ b/src/runner/PlutoRunner.jl @@ -448,7 +448,7 @@ function run_expression(m::Module, expr::Any, cell_id::UUID, function_wrapped_in try computer = register_computer(expr, expr_id, cell_id, collect.(function_wrapped_info)...) catch e - # @error "Failed to generate computer function" expr exception=(e,stacktrace(catch_backtrace())) + @error "Failed to generate computer function" expr exception=(e,stacktrace(catch_backtrace())) return run_expression(m, expr, cell_id, nothing) end end @@ -457,7 +457,7 @@ function run_expression(m::Module, expr::Any, cell_id::UUID, function_wrapped_in # The fix is to detect this situation and run the expression in the classical way. ans, runtime = if any(name -> !isdefined(m, name), computer.input_globals) # Do run_expression but with function_wrapped_info=nothing so it doesn't go in a Computer() - # @warn "Got variables that don't exist, running outside of computer" not_existing=filter(name -> !isdefined(m, name), computer.input_globals) + @warn "Got variables that don't exist, running outside of computer" not_existing=filter(name -> !isdefined(m, name), computer.input_globals) run_expression(m, expr, cell_id, nothing) else run_inside_trycatch(m, () -> compute(m, computer), computer.return_proof) From d673ceea6dc5a539b599e38986377da623c76c4c Mon Sep 17 00:00:00 2001 From: Michiel Dral Date: Sat, 30 Oct 2021 00:44:22 +0000 Subject: [PATCH 20/28] My bad this really is the last --- src/runner/PlutoRunner.jl | 4 ++-- test/MacroAnalysis.jl | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/runner/PlutoRunner.jl b/src/runner/PlutoRunner.jl index 763d51a40b..ae5f8279a8 100644 --- a/src/runner/PlutoRunner.jl +++ b/src/runner/PlutoRunner.jl @@ -448,7 +448,7 @@ function run_expression(m::Module, expr::Any, cell_id::UUID, function_wrapped_in try computer = register_computer(expr, expr_id, cell_id, collect.(function_wrapped_info)...) catch e - @error "Failed to generate computer function" expr exception=(e,stacktrace(catch_backtrace())) + # @error "Failed to generate computer function" expr exception=(e,stacktrace(catch_backtrace())) return run_expression(m, expr, cell_id, nothing) end end @@ -457,7 +457,7 @@ function run_expression(m::Module, expr::Any, cell_id::UUID, function_wrapped_in # The fix is to detect this situation and run the expression in the classical way. ans, runtime = if any(name -> !isdefined(m, name), computer.input_globals) # Do run_expression but with function_wrapped_info=nothing so it doesn't go in a Computer() - @warn "Got variables that don't exist, running outside of computer" not_existing=filter(name -> !isdefined(m, name), computer.input_globals) + # @warn "Got variables that don't exist, running outside of computer" not_existing=filter(name -> !isdefined(m, name), computer.input_globals) run_expression(m, expr, cell_id, nothing) else run_inside_trycatch(m, () -> compute(m, computer), computer.return_proof) diff --git a/test/MacroAnalysis.jl b/test/MacroAnalysis.jl index 58d5c9b102..4f4c534663 100644 --- a/test/MacroAnalysis.jl +++ b/test/MacroAnalysis.jl @@ -704,7 +704,6 @@ import Pluto: PlutoRunner, Notebook, WorkspaceManager, Cell, ServerSession, Clie @test ":hello" == cell(4).output.body @test :b ∈ notebook.topology.nodes[cell(3)].definitions @test [:c, Symbol("@my_assign")] βŠ† notebook.topology.nodes[cell(3)].references - @test cell(3).cell_dependencies.contains_user_defined_macrocalls == true setcode(notebook.cells[2], "c = :world") update_run!(🍭, notebook, cell(2)) From 80c2af2ee6f5010be6b86db8ebc98c013fe63c43 Mon Sep 17 00:00:00 2001 From: Michiel Dral Date: Sat, 30 Oct 2021 18:46:11 +0200 Subject: [PATCH 21/28] GiveMeCellID!! --- src/evaluation/Run.jl | 7 +++- src/runner/PlutoRunner.jl | 82 +++++++++++++++++++++------------------ 2 files changed, 51 insertions(+), 38 deletions(-) diff --git a/src/evaluation/Run.jl b/src/evaluation/Run.jl index 6aab42990f..25771f6621 100644 --- a/src/evaluation/Run.jl +++ b/src/evaluation/Run.jl @@ -326,7 +326,12 @@ function resolve_topology( push!(still_unresolved_nodes, cell) end - result = analyze_macrocell(cell) + result = try + analyze_macrocell(cell) + catch error + @error "Macro call expansion failed with a non-macroexpand error" error + Failure(error) + end if result isa Success (new_node, function_wrapped, forced_expr_id) = result.result union!(new_node.macrocalls, unresolved_topology.nodes[cell].macrocalls) diff --git a/src/runner/PlutoRunner.jl b/src/runner/PlutoRunner.jl index ae5f8279a8..83264ebdd8 100644 --- a/src/runner/PlutoRunner.jl +++ b/src/runner/PlutoRunner.jl @@ -37,7 +37,7 @@ const cell_expanded_exprs = Dict{UUID,CachedMacroExpansion}() - +struct GiveMeCellID end @@ -117,18 +117,24 @@ end no_workspace_ref(expr::Expr, mod_name=nothing) = Expr(expr.head, map(arg -> no_workspace_ref(arg, mod_name), expr.args)...) no_workspace_ref(other, _=nothing) = other + +replace_pluto_properties_in_expr(::GiveMeCellID; cell_id) = cell_id +replace_pluto_properties_in_expr(expr::Expr; cell_id) = Expr(expr.head, map(arg -> replace_pluto_properties_in_expr(arg, cell_id=cell_id), expr.args)...) +replace_pluto_properties_in_expr(other; cell_id) = other + + function sanitize_expr(symbol::Symbol) symbol end -function sanitize_expr(dt::Union{DataType,Enum}) - Symbol(dt) -end +# function sanitize_expr(dt::Union{DataType,Enum}) +# Symbol(dt) +# end function sanitize_expr(ref::GlobalRef) test_mod_name = nameof(ref.mod) |> string if startswith(test_mod_name, "workspace#") - ref.name + sanitize_expr(ref.name) else wrap_dot(ref) end @@ -140,54 +146,55 @@ end # a function as part of an Expr is most likely a closure # returned from a macro -function sanitize_expr(func::Function) - mt = typeof(func).name.mt - GlobalRef(mt.module, mt.name) |> sanitize_expr -end +# function sanitize_expr(func::Function) +# mt = typeof(func).name.mt +# GlobalRef(mt.module, mt.name) |> sanitize_expr +# end -function sanitize_expr(union_all::UnionAll) - sanitize_expr(union_all.body) -end +# function sanitize_expr(union_all::UnionAll) +# sanitize_expr(union_all.body) +# end -function sanitize_expr(vec::AbstractVector) - Expr(:vect, sanitize_value.(vec)...) -end +# function sanitize_expr(vec::AbstractVector) +# Expr(:vect, sanitize_value.(vec)...) +# end -function sanitize_expr(tuple::Tuple) - Expr(:tuple, sanitize_value.(tuple)...) -end +# function sanitize_expr(tuple::Tuple) +# Expr(:tuple, sanitize_value.(tuple)...) +# end -function sanitize_expr(dict::Dict) - Expr(:call, :Dict, (sanitize_value(pair) for pair in dict)...) -end +# function sanitize_expr(dict::Dict) +# Expr(:call, :Dict, (sanitize_value(pair) for pair in dict)...) +# end -function sanitize_expr(pair::Pair) - Expr(:call, :(=>), sanitize_value(pair.first), sanitize_value(pair.second)) -end +# function sanitize_expr(pair::Pair) +# Expr(:call, :(=>), sanitize_value(pair.first), sanitize_value(pair.second)) +# end -function sanitize_expr(set::Set) - Expr(:call, :Set, Expr(:vect, sanitize_value.(set)...)) -end +# function sanitize_expr(set::Set) +# Expr(:call, :Set, Expr(:vect, sanitize_value.(set)...)) +# end -function sanitize_expr(mod::Module) - fullname(mod) |> wrap_dot -end +# function sanitize_expr(mod::Module) +# fullname(mod) |> wrap_dot +# end # An instanciation of a struct as part of an Expr # will not de-serializable in the Pluto process, only send if it is a child of PlutoRunner, Base or Core function sanitize_expr(other) - typename = other |> typeof - typename |> parentmodule |> Symbol ∈ [:Core, :PlutoRunner, :Base] ? - other : - Symbol(typename) + # typename = other |> typeof + # typename |> parentmodule |> Symbol ∈ [:Core, :PlutoRunner, :Base] ? + # other : + # Symbol(typename) + nothing end # A vector of Symbols need to be serialized as QuoteNode(sym) -sanitize_value(sym::Symbol) = QuoteNode(sym) +# sanitize_value(sym::Symbol) = QuoteNode(sym) -sanitize_value(ex::Expr) = Expr(:quote, ex) +# sanitize_value(ex::Expr) = Expr(:quote, ex) -sanitize_value(other) = sanitize_expr(other) +# sanitize_value(other) = sanitize_expr(other) function try_macroexpand(mod, cell_uuid, expr) @@ -198,6 +205,7 @@ function try_macroexpand(mod, cell_uuid, expr) # Removes baked in references to the module this was macroexpanded in. # Fix for https://github.com/fonsp/Pluto.jl/issues/1112 expr_to_save = no_workspace_ref(expanded_expr) + expr_to_save = replace_pluto_properties_in_expr(expanded_expr, cell_id=cell_uuid) cell_expanded_exprs[cell_uuid] = CachedMacroExpansion( original_expr_hash=expr_hash(expr), From f09c51b8e0cac24ad802480883b767f8e191c3bb Mon Sep 17 00:00:00 2001 From: Michiel Dral Date: Mon, 1 Nov 2021 02:29:00 +0100 Subject: [PATCH 22/28] Make sanitize_expr stricter because I can --- src/runner/PlutoRunner.jl | 52 ++++++--------------------------------- 1 file changed, 8 insertions(+), 44 deletions(-) diff --git a/src/runner/PlutoRunner.jl b/src/runner/PlutoRunner.jl index 83264ebdd8..8639152a97 100644 --- a/src/runner/PlutoRunner.jl +++ b/src/runner/PlutoRunner.jl @@ -144,50 +144,11 @@ function sanitize_expr(expr::Expr) Expr(expr.head, sanitize_expr.(expr.args)...) end -# a function as part of an Expr is most likely a closure -# returned from a macro -# function sanitize_expr(func::Function) -# mt = typeof(func).name.mt -# GlobalRef(mt.module, mt.name) |> sanitize_expr -# end - -# function sanitize_expr(union_all::UnionAll) -# sanitize_expr(union_all.body) -# end - -# function sanitize_expr(vec::AbstractVector) -# Expr(:vect, sanitize_value.(vec)...) -# end - -# function sanitize_expr(tuple::Tuple) -# Expr(:tuple, sanitize_value.(tuple)...) -# end - -# function sanitize_expr(dict::Dict) -# Expr(:call, :Dict, (sanitize_value(pair) for pair in dict)...) -# end - -# function sanitize_expr(pair::Pair) -# Expr(:call, :(=>), sanitize_value(pair.first), sanitize_value(pair.second)) -# end - -# function sanitize_expr(set::Set) -# Expr(:call, :Set, Expr(:vect, sanitize_value.(set)...)) -# end - -# function sanitize_expr(mod::Module) -# fullname(mod) |> wrap_dot -# end - -# An instanciation of a struct as part of an Expr -# will not de-serializable in the Pluto process, only send if it is a child of PlutoRunner, Base or Core -function sanitize_expr(other) - # typename = other |> typeof - # typename |> parentmodule |> Symbol ∈ [:Core, :PlutoRunner, :Base] ? - # other : - # Symbol(typename) - nothing -end +sanitize_expr(linenumbernode::LineNumberNode) = linenumbernode +sanitize_expr(bool::Bool) = bool +# In all cases of more complex objects, we just don't send it. +# It's not like the expression explorer will look into them at all. +sanitize_expr(other) = nothing # A vector of Symbols need to be serialized as QuoteNode(sym) # sanitize_value(sym::Symbol) = QuoteNode(sym) @@ -297,6 +258,9 @@ parse_cell_id(filename::AbstractString) = match(r"#==#(.*)", filename).captures |> only |> UUID function register_cleanup(f::Function, cell_id::UUID) + # TODO Don't need this, everything is "function wrapped" for hook purposes, + # .... because of the cached macro expansion.. This does however mean we need + # .... a separate store for cleanup functions @assert haskey(computers, cell_id) "The cell $cell_id is not function wrapped" push!(computers[cell_id].cleanup_funcs, f) nothing From 77067f74c235eb1647ab5d564303b516757b188a Mon Sep 17 00:00:00 2001 From: Michiel Dral Date: Tue, 2 Nov 2021 15:43:58 +0100 Subject: [PATCH 23/28] Getting started --- src/Pluto.jl | 1 + src/evaluation/ChildProcesses.jl | 5 + src/evaluation/ChildProcessesNotebook.jl | 752 +++++++++++++++++++++++ src/evaluation/ChildProcessesTest.jl | 459 ++++++++++++++ src/evaluation/Run.jl | 2 +- src/evaluation/WorkspaceManager.jl | 160 ++--- src/webserver/REPLTools.jl | 8 +- 7 files changed, 1313 insertions(+), 74 deletions(-) create mode 100644 src/evaluation/ChildProcesses.jl create mode 100644 src/evaluation/ChildProcessesNotebook.jl create mode 100644 src/evaluation/ChildProcessesTest.jl diff --git a/src/Pluto.jl b/src/Pluto.jl index 66d19a0949..763217b1eb 100644 --- a/src/Pluto.jl +++ b/src/Pluto.jl @@ -21,6 +21,7 @@ const JULIA_VERSION_STR = 'v' * string(VERSION) include("./notebook/PathHelpers.jl") include("./notebook/Export.jl") include("./Configuration.jl") +include("./evaluation/ChildProcesses.jl") include("./evaluation/Tokens.jl") include("./runner/PlutoRunner.jl") diff --git a/src/evaluation/ChildProcesses.jl b/src/evaluation/ChildProcesses.jl new file mode 100644 index 0000000000..828fb0c4d2 --- /dev/null +++ b/src/evaluation/ChildProcesses.jl @@ -0,0 +1,5 @@ +ChildProcesses = @eval Main module ChildProcesses + +include("./ChildProcessesNotebook.jl") + +end \ No newline at end of file diff --git a/src/evaluation/ChildProcessesNotebook.jl b/src/evaluation/ChildProcessesNotebook.jl new file mode 100644 index 0000000000..ee920e904b --- /dev/null +++ b/src/evaluation/ChildProcessesNotebook.jl @@ -0,0 +1,752 @@ +### A Pluto.jl notebook ### +# v0.17.0 + +using Markdown +using InteractiveUtils + +# ╔═║ 6925ffbb-fd9e-402c-a483-c78f28f892a5 +import Serialization + +# ╔═║ 8c97d5cb-e346-4b94-b2a1-4fb5ff093bd5 +import UUIDs: UUID, uuid4 + +# ╔═║ a6e50946-dd3d-4738-adc1-26534e184776 +md""" +## Binary message frame thing + +Functions to send and read binary messages over a stable connection, also functions to serialize to a bytearray. + +Pretty simple right now, might want to add cooler stuff later. +""" + +# ╔═║ 131a123b-e319-4939-90bf-7fb035ab2e75 +MessageLength = Int + +# ╔═║ e393ff80-3995-11ec-1217-b3642a509067 +function read_message(stream) + message_length_buffer = Vector{UInt8}(undef, 8) + bytesread = readbytes!(stream, message_length_buffer, 8) + how_long_will_the_message_be = reinterpret(MessageLength, message_length_buffer)[1] + + message_buffer = Vector{UInt8}(undef, how_long_will_the_message_be) + _bytesread = readbytes!(stream, message_buffer, how_long_will_the_message_be) + + message_buffer +end + +# ╔═║ 52495855-a52d-4edf-9c7c-811a5060e641 +function to_binary(message) + io = PipeBuffer() + serialized = Serialization.serialize(io, message) + read(io) +end + +# ╔═║ 7da416d1-5dfb-4510-9d32-62c1464a83d4 +function from_binary(message) + Serialization.deserialize(IOBuffer(message)) +end + +# ╔═║ 87612815-7981-4a69-a65b-4465f48c9fd9 +struct ReadMessages + io::IO +end + +# ╔═║ 009ad714-b759-420a-b49a-6caed7ee3faf +function Base.iterate(message_reader::ReadMessages, state=nothing) + @warn "Iterating!" + if eof(message_reader.io) + @warn "Pfff" + return nothing + end + @warn "Nice" + + message = from_binary(read_message(message_reader.io)) + (message, nothing) +end + +# ╔═║ faf3c68e-d0fb-4dd9-ac1b-1ff8d922134b +function send_message(stream, bytes) + # bytes = Vector{UInt8}(message) + message_length = convert(MessageLength, length(bytes)) + # @info "message_length" message_length bytes + how_long_will_the_message_be = reinterpret(UInt8, [message_length]) + + @warn "Writing to stream!!!" + _1 = write(stream, how_long_will_the_message_be) + _2 = write(stream, bytes) + @warn "WROTE TO STREAM" + + _1 + _2 + @info "Write" bytesavailable(stream.in) _1 _2 +end + +# ╔═║ de680a32-e053-4655-a1cd-111d81a037e6 +md""" +## Child process side +""" + +# ╔═║ ce2acdd1-b4bb-4a8f-8346-a56dfa55f56b +Base.@kwdef mutable struct ParentProcess + parent_to_child=stdin + child_to_parent=stdout + channels::Dict{UUID,AbstractChannel}=Dict{UUID,AbstractChannel}() + lock=ReentrantLock() +end + +# ╔═║ 87182605-f5a1-46a7-968f-bdfb9d0e08fa +function setup_parent_process() + real_stdout = stdout + # Redirect things written to stdout to stderr, + # stderr will showup in the Pluto with_terminal view. + redirect_stdout(stderr) + + # TODO Maybe redirect stdin too, just to be sure? + + ParentProcess( + parent_to_child=stdin, + child_to_parent=real_stdout, + ) +end + +# ╔═║ b05be9c7-8cc1-47f9-8baa-db66ac83c24f +Base.@kwdef struct ParentChannel + process::ParentProcess + channel_id::UUID + result_channel::AbstractChannel +end + +# ╔═║ 17cb5a65-2a72-4e7d-8fba-901452b2c19f +md""" +## Parent process side +""" + +# ╔═║ 85920c27-d8a6-4b5c-93b6-41daa5866f9d +Base.@kwdef mutable struct ChildProcess + process::Base.AbstractPipe + channels::Dict{UUID,AbstractChannel}=Dict{UUID,AbstractChannel}() + lock=ReentrantLock() +end + +# ╔═║ 63531683-e295-4ba6-811f-63b0d384ba0f +Base.@kwdef struct ChildProcessException <: Exception + process::ChildProcess + captured::CapturedException +end + +# ╔═║ e6b3d1d9-0245-4dc8-a0a5-5831c254479b +Base.@kwdef struct ProcessExitedException <: Exception + process::ChildProcess +end + +# ╔═║ af4f0720-f7f7-4d8a-bd8c-8cf5abaf10a0 +Base.kill(process::ChildProcess) = Base.kill(process.process) + +# ╔═║ d4da756e-fa81-49d8-8c39-d76f9d15e96f +Base.process_running(p::ChildProcess) = Base.process_running(p.process) + +# ╔═║ e787d8bd-499d-4a41-8913-62b3d9346748 +Base.wait(process::ChildProcess) = Base.wait(process.process) + +# ╔═║ 3a62b419-eca2-4de1-89a4-fc9ad6f68372 +struct SingleTakeChannel end + +# ╔═║ 61410bd1-8456-4608-92d9-3c863f65b89c +function Base.take!(::SingleTakeChannel) + nothing +end + +# ╔═║ 409707ff-b1ea-453d-b933-0a4b1e5f44c8 +function Base.iterate(::SingleTakeChannel, _=nothing) + (nothing, nothing) +end + +# ╔═║ 0feb758e-38d4-48c0-a5e9-9d1129b1b1b2 +function Base.isopen(::SingleTakeChannel) + true +end + +# ╔═║ abe897ff-6e86-40eb-a1a6-2918b6c3c5a7 +struct LocalSandbox end + +# ╔═║ 934d18d4-936e-4ee0-aa6e-86aa6f66774c +juliapath() = joinpath(Sys.BINDIR::String, Base.julia_exename()) + +# ╔═║ b48f36d1-e3d2-408e-a705-93e609f2fd04 +fieldnames(Base.Process) + +# ╔═║ 884e103a-8925-477d-a264-f15d02a49aa9 +md""" +## ChildChannel + +I guess this is similar to Distributed.ChildChannel. +It communicates all actions that happen on it (take!, put!, close) to the other side. There is a local channel stored that is used as buffer. +""" + +# ╔═║ b8865d63-19fa-4438-87a2-ccb531bd73a4 +Base.@kwdef struct ChildChannel{T} <: AbstractChannel{T} + process::ChildProcess + id::UUID=uuid4() + local_channel::AbstractChannel{T} +end + +# ╔═║ fc0ca4b5-e03b-4c08-b43d-913ee12269c7 +abstract type Message end + +# ╔═║ adfb2d12-07a0-4d5e-8e24-1676c07107c7 +struct CreateChildChannelMessage <: Message + expr +end + +# ╔═║ e8256f1f-c778-4fb3-a2d3-12da0e1cb3da +struct BaseMessage <: Message + value::Any +end + +# ╔═║ f9270f05-fa22-4027-a2ca-7db61d057c56 +struct ErrorMessage <: Message + error::Exception +end + +# ╔═║ 0ba318bc-7b4c-4d0f-a46c-2f9dca756926 +struct ChannelPushMessage <: Message + value::Any +end + +# ╔═║ c83f2936-3768-4724-9c5c-335d7a4aae03 +struct ChannelCloseMessage <: Message +end + +# ╔═║ 441c27e4-6884-4887-9cd5-bc55d0c49760 +struct ChannelTakeMessage <: Message +end + +# ╔═║ f81ff6f7-f230-4373-b60b-76e8a0eba929 +Base.@kwdef struct Envelope + channel_id::UUID + message::Message +end + +# ╔═║ 590a7882-3d69-48b0-bb1b-f476c7f8a885 +function respond_to_parent(; to::ParentProcess, about::UUID, with::Message) + binary_message = to_binary(Envelope( + channel_id=about, + message=with, + )) + lock(to.lock) + try + # @info "Message out" about message=typeof(with) + send_message(to.child_to_parent, binary_message) + finally + unlock(to.lock) + end + +end + +# ╔═║ 20f66652-bca0-4e47-a4b7-502cfbcb3db5 +function send_message_without_response(process::ChildProcess, envelope::Envelope) + if !process_running(process) + throw(ProcessExitedException(process=process)) + end + + lock(process.lock) + try + @info "Message out" message=typeof(envelope.message) + send_message(process.process, to_binary(envelope)) + return envelope + finally + unlock(process.lock) + end +end + +# ╔═║ e1e79669-8d0a-4b04-a7fd-469e7d8e65b1 +function Base.take!(channel::ChildChannel) + if !isopen(channel.local_channel) + # Trigger InvalidStateException + eval(:(@error "ChildChannel closed D:")) + take!(channel.local_channel) + end + + send_message_without_response(channel.process, Envelope( + channel_id=channel.id, + message=ChannelTakeMessage(), + )) + take!(channel.local_channel) +end + +# ╔═║ d3ce3ecf-c1e4-4484-ac38-b332a4b13034 +begin + import Base: close + close(process::ChildProcess) = close(process.process) + function close(channel::ChildChannel) + if isopen(channel.local_channel) + try + close(channel.local_channel) + if process_running(channel.process.process) + send_message_without_response(channel.process, Envelope( + channel_id=channel.id, + message=ChannelCloseMessage() + )) + end + catch e + @error "Problem with closing?" e stack=stacktrace(catch_backtrace()) + nothing + end + end + delete!(channel.process.channels, channel.id) + end + function close(::SingleTakeChannel) end +end + +# ╔═║ 6a7aa0ce-0ae3-4e7d-a93a-bfdebe406220 +begin + function handle_child_message(; + message, + process::ChildProcess, + input_channel::AbstractChannel, + output_channel::AbstractChannel, + ) + @warn "Unknown message type" message + end + + function handle_child_message(; + message::ChannelPushMessage, + process::ChildProcess, + input_channel::AbstractChannel, + output_channel::AbstractChannel, + ) + put!(output_channel, message.value) + end + + function handle_child_message(; + message::ChannelCloseMessage, + process::ChildProcess, + input_channel::AbstractChannel, + output_channel::AbstractChannel, + ) + close(output_channel) + end + + function handle_child_message(; + message::ErrorMessage, + process::ChildProcess, + input_channel::AbstractChannel, + output_channel::AbstractChannel, + ) + close(input_channel, ChildProcessException( + process=process, + captured=message.error, + )) + end +end + +# ╔═║ f99f4659-f71e-4f2e-a674-67ba69289817 +function create_channel(process::ChildProcess, expr) + channel_id = uuid4() + envelope = send_message_without_response(process, Envelope( + channel_id=channel_id, + message=CreateChildChannelMessage(expr) + )) + + response_channel = Channel{Message}() + process.channels[channel_id] = response_channel + + @info "CREATING CHANNEL" + actual_values_channes = Channel(Inf) do ch + try + for message in response_channel + @info "MESSAGE FROM CHILD CHANNEL" channel_id message + handle_child_message( + message=message, + process=process, + input_channel=response_channel, + output_channel=ch, + ) + end + catch e + @error "ERROR???" e + close(ch, e) + finally + close(ch) + close(response_channel) + end + end + + ChildChannel( + process=process, + id=channel_id, + local_channel=actual_values_channes, + ) +end + +# ╔═║ 2342d663-030f-4ed2-b1d5-5be1910b6d4c +function create_channel(fn, process::ChildProcess, message) + remote_channel = create_channel(process, message) + + try + return fn(remote_channel) + finally + close(remote_channel) + end +end + +# ╔═║ 4b42e233-1f06-49c9-8c6a-9dc21c21ffb7 +function call(process::ChildProcess, expr) + create_channel(process, quote + Channel() do ch + put!(ch, $expr) + end + end) do channel + @info "Taking" + take!(channel) + end +end + +# ╔═║ e3e16a8b-7124-4678-8fe7-12ed449e1954 +function call_without_fetch(process::ChildProcess, expr) + create_channel(process, quote + $expr + $(SingleTakeChannel()) + end) do channel + take!(channel) + end +end + +# ╔═║ 54a03cba-6d00-4632-8bd6-c60753c15ae6 +function start_from_child_loop(child_process) + try + for result in ReadMessages(child_process.process) + if !(result isa Envelope && result.channel_id !== nothing) + throw("Huh") + end + + @warn "MESSAGE FROM CHILD" result.message + + if haskey(child_process.channels, result.channel_id) + message = result.message + channel = child_process.channels[result.channel_id] + put!(channel, message) + else + @error "No channel to push to" + throw("No channel to push to") + end + end + catch error + @error "Error in main processing" error bt=stacktrace(catch_backtrace()) + close(child_process.process) + finally + for (channel_id, channel) in collect(child_process.channels) + close(channel, ProcessExitedException(process=child_process)) + end + end +end + +# ╔═║ d3fe8144-6ba8-4dd9-b0e3-941a96422267 +function create_child_process(; custom_stderr=stderr, exeflags=["--history-file=no"]) + this_file = split(@__FILE__(), "#")[1] + this_module = string(nameof(@__MODULE__)) + + code = """ + # Make sure this library is included in main + var"$this_module" = @eval Main module var"$this_module" + include("$this_file") + end + + @info "Booted up!" + + const parent_process = var"$this_module".setup_parent_process() + + try + @info "Startin..." + var"$this_module".listen_for_messages_from_parent(parent_process) + @info "Done?" + catch error + @error "Shutdown error" error + rethrow(error) + end + """ + + process = open( + pipeline(`$(juliapath()) $exeflags -e $code`, stderr=custom_stderr), + read=true, + write=true, + ) + + child_process = ChildProcess(process=process) + schedule(Task() do + start_from_child_loop(child_process) + end) + child_process +end + +# ╔═║ 5fb65aec-3512-4f48-98ce-300ab9fdadfe +begin + function Base.iterate(channel::ChildChannel) + send_message_without_response(channel.process, Envelope( + channel_id=channel.id, + message=ChannelTakeMessage(), + )) + Base.iterate(channel.local_channel) + end + function Base.iterate(channel::ChildChannel, x) + send_message_without_response(channel.process, Envelope( + channel_id=channel.id, + message=ChannelTakeMessage(), + )) + Base.iterate(channel.local_channel, x) + end +end + +# ╔═║ 46d905fc-d41e-4c9a-a808-14710f64293a +begin + function handle_message_from_parent_channel(; + parent_channel::ParentChannel, + message::ChannelTakeMessage, + ) + @info "Channel message taken!" + next = iterate(parent_channel.result_channel) + if next === nothing + respond_to_parent( + to=parent_channel.process, + about=parent_channel.channel_id, + with=ChannelCloseMessage(), + ) + true + else + respond_to_parent( + to=parent_channel.process, + about=parent_channel.channel_id, + with=ChannelPushMessage(next[1]) + ) + false + end + end + + function handle_message_from_parent_channel(; + parent_channel::ParentChannel, + message::ChannelCloseMessage, + ) + if isopen(parent_channel.result_channel) + close(parent_channel.result_channel) + end + false + end + + function handle_message_from_parent_channel(; + parent_channel::ParentChannel, + message::Message, + ) + @warn "Unknown channel message type" message + false + end +end + +# ╔═║ 13089c5d-f833-4fb7-b8cd-4158b1a57103 +function create_parent_channel(; process, channel_id, result_channel) + parent_channel = ParentChannel( + process=process, + channel_id=channel_id, + result_channel=result_channel, + ) + Channel(Inf) do input_channel + try + for message in input_channel + @info "Applying message" message + @timed if handle_message_from_parent_channel( + parent_channel=parent_channel, + message=message, + ) + break + end + end + catch error + if error isa InterruptException + rethrow(error) + else + @error "Error in channel" error + respond_to_parent( + to=process, + about=channel_id, + with=ErrorMessage(CapturedException(error, catch_backtrace())) + ) + end + finally + close(input_channel) + end + end +end + +# ╔═║ 38ca1c5b-e27e-4458-98a7-786b2d302ba3 +begin + function handle_message_from_parent(; + process::ParentProcess, + channel_id::UUID, + message + ) + @warn "Unknown message type from parent" message + end + + function handle_message_from_parent(; + process::ParentProcess, + channel_id::UUID, + message::CreateChildChannelMessage + ) + result_channel = Main.eval(message.expr) + channels[channel_id] = create_parent_channel( + process=process, + channel_id=channel_id, + result_channel=result_channel, + ) + end + + function handle_message_from_parent(; + process::ParentProcess, + channel_id::UUID, + message::ChannelCloseMessage + ) + if haskey(process.channels, channel_id) && isopen(process.channels[channel_id]) + put!(process.channels[channel_id], message) + end + end + + function handle_message_from_parent(; + process::ParentProcess, + channel_id::UUID, + message::ChannelTakeMessage + ) + if haskey(process.channels, channel_id) && isopen(process.channels[channel_id]) + put!(process.channels[channel_id], message) + else + respond_to_parent( + to=process, + about=channel_id, + with=ChannelCloseMessage(), + ) + end + end +end + +# ╔═║ e4109311-8252-4793-87b8-eae807df7997 +function listen_for_messages_from_parent(parent_process::ParentProcess) + channels = parent_process.channels + + locks = Dict{UUID, ReentrantLock}() + + @info "Well..." parent_process.parent_to_child + for envelope in ReadMessages(parent_process.parent_to_child) + @info "Message from parent!" envelope + channel_lock = get!(locks, envelope.channel_id) do + ReentrantLock() + end + schedule(Task() do + lock(channel_lock) + try + handle_message_from_parent( + process=parent_process, + channel_id=envelope.channel_id, + message=envelope.message, + ) + catch error + respond_to_parent( + to=parent_process, + about=envelope.channel_id, + with=ErrorMessage(CapturedException(error, catch_backtrace())) + ) + finally + unlock(channel_lock) + end + end) + end + @warn "oh no" +end + +# ╔═║ 1f868b3c-df9a-4d4d-a6a9-9a196298a3af +md"---" + +# ╔═║ bd530d27-ad17-45ef-8d8b-4b9e956f438d + + +# ╔═║ 00000000-0000-0000-0000-000000000001 +PLUTO_PROJECT_TOML_CONTENTS = """ +[deps] +Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +""" + +# ╔═║ 00000000-0000-0000-0000-000000000002 +PLUTO_MANIFEST_TOML_CONTENTS = """ +# This file is machine-generated - editing it directly is not advised + +[[Random]] +deps = ["Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" + +[[Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +""" + +# ╔═║ Cell order: +# ╠═6925ffbb-fd9e-402c-a483-c78f28f892a5 +# ╠═8c97d5cb-e346-4b94-b2a1-4fb5ff093bd5 +# ╠═63531683-e295-4ba6-811f-63b0d384ba0f +# ╠═e6b3d1d9-0245-4dc8-a0a5-5831c254479b +# β•Ÿβ”€a6e50946-dd3d-4738-adc1-26534e184776 +# ╠═131a123b-e319-4939-90bf-7fb035ab2e75 +# ╠═e393ff80-3995-11ec-1217-b3642a509067 +# ╠═faf3c68e-d0fb-4dd9-ac1b-1ff8d922134b +# β•Ÿβ”€52495855-a52d-4edf-9c7c-811a5060e641 +# β•Ÿβ”€7da416d1-5dfb-4510-9d32-62c1464a83d4 +# β•Ÿβ”€87612815-7981-4a69-a65b-4465f48c9fd9 +# ╠═009ad714-b759-420a-b49a-6caed7ee3faf +# β•Ÿβ”€de680a32-e053-4655-a1cd-111d81a037e6 +# ╠═ce2acdd1-b4bb-4a8f-8346-a56dfa55f56b +# ╠═87182605-f5a1-46a7-968f-bdfb9d0e08fa +# ╠═b05be9c7-8cc1-47f9-8baa-db66ac83c24f +# ╠═590a7882-3d69-48b0-bb1b-f476c7f8a885 +# ╠═46d905fc-d41e-4c9a-a808-14710f64293a +# ╠═13089c5d-f833-4fb7-b8cd-4158b1a57103 +# ╠═38ca1c5b-e27e-4458-98a7-786b2d302ba3 +# ╠═e4109311-8252-4793-87b8-eae807df7997 +# β•Ÿβ”€17cb5a65-2a72-4e7d-8fba-901452b2c19f +# ╠═85920c27-d8a6-4b5c-93b6-41daa5866f9d +# ╠═af4f0720-f7f7-4d8a-bd8c-8cf5abaf10a0 +# ╠═d4da756e-fa81-49d8-8c39-d76f9d15e96f +# ╠═e787d8bd-499d-4a41-8913-62b3d9346748 +# ╠═20f66652-bca0-4e47-a4b7-502cfbcb3db5 +# ╠═6a7aa0ce-0ae3-4e7d-a93a-bfdebe406220 +# ╠═f99f4659-f71e-4f2e-a674-67ba69289817 +# ╠═3a62b419-eca2-4de1-89a4-fc9ad6f68372 +# ╠═61410bd1-8456-4608-92d9-3c863f65b89c +# ╠═409707ff-b1ea-453d-b933-0a4b1e5f44c8 +# ╠═0feb758e-38d4-48c0-a5e9-9d1129b1b1b2 +# ╠═2342d663-030f-4ed2-b1d5-5be1910b6d4c +# ╠═abe897ff-6e86-40eb-a1a6-2918b6c3c5a7 +# ╠═4b42e233-1f06-49c9-8c6a-9dc21c21ffb7 +# ╠═e3e16a8b-7124-4678-8fe7-12ed449e1954 +# ╠═934d18d4-936e-4ee0-aa6e-86aa6f66774c +# ╠═b48f36d1-e3d2-408e-a705-93e609f2fd04 +# ╠═54a03cba-6d00-4632-8bd6-c60753c15ae6 +# ╠═d3fe8144-6ba8-4dd9-b0e3-941a96422267 +# β•Ÿβ”€884e103a-8925-477d-a264-f15d02a49aa9 +# ╠═b8865d63-19fa-4438-87a2-ccb531bd73a4 +# ╠═e1e79669-8d0a-4b04-a7fd-469e7d8e65b1 +# ╠═d3ce3ecf-c1e4-4484-ac38-b332a4b13034 +# ╠═5fb65aec-3512-4f48-98ce-300ab9fdadfe +# ╠═fc0ca4b5-e03b-4c08-b43d-913ee12269c7 +# ╠═adfb2d12-07a0-4d5e-8e24-1676c07107c7 +# ╠═e8256f1f-c778-4fb3-a2d3-12da0e1cb3da +# ╠═f9270f05-fa22-4027-a2ca-7db61d057c56 +# ╠═0ba318bc-7b4c-4d0f-a46c-2f9dca756926 +# ╠═c83f2936-3768-4724-9c5c-335d7a4aae03 +# ╠═441c27e4-6884-4887-9cd5-bc55d0c49760 +# ╠═f81ff6f7-f230-4373-b60b-76e8a0eba929 +# β•Ÿβ”€1f868b3c-df9a-4d4d-a6a9-9a196298a3af +# ╠═bd530d27-ad17-45ef-8d8b-4b9e956f438d +# β•Ÿβ”€00000000-0000-0000-0000-000000000001 +# β•Ÿβ”€00000000-0000-0000-0000-000000000002 diff --git a/src/evaluation/ChildProcessesTest.jl b/src/evaluation/ChildProcessesTest.jl new file mode 100644 index 0000000000..5239204b41 --- /dev/null +++ b/src/evaluation/ChildProcessesTest.jl @@ -0,0 +1,459 @@ +### A Pluto.jl notebook ### +# v0.17.0 + +using Markdown +using InteractiveUtils + +# This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error). +macro bind(def, element) + quote + local el = $(esc(element)) + global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : missing + el + end +end + +# ╔═║ 7e613bd2-616a-4687-8af5-a22c7a747d97 +import Serialization + +# ╔═║ d30e1e1b-fa6f-4fbc-bccb-1fcfc7b829df +import UUIDs: UUID, uuid4 + +# ╔═║ a901d255-c18c-45ed-9827-afd79246613c +module PlutoHooks include("./PlutoHooks.jl") end + +# ╔═║ f7d14367-27d7-41a5-9f6a-79cf5e721a7d +import PlutoUI + +# ╔═║ f9675ee0-e728-420b-81bd-22e57583c587 +import .PlutoHooks: @use_effect, @use_ref, @use_state, @background, @use_memo + +# ╔═║ ca96e0d5-0904-4ae5-89d0-c1a9187710a1 +Base.@kwdef struct PlutoProcess + process + stdin + stdout + stderr +end + +# ╔═║ b7734627-ba2a-48d8-8fd8-a5c94716da20 +function Base.show(io::IO, ::MIME"text/plain", process::PlutoProcess) + write(io, process_running(process.process) ? "Running" : "Stopped") +end + +# ╔═║ c9a9a089-037b-4fa8-91b2-980a318faee7 +begin + Frames = PlutoHooks.@ingredients("./SerializationLibrary.jl") + ChildProcesses = Frames + Main.eval(quote + var"SerializationLibrary.jl" = $(Frames) + end) +end + +# ╔═║ 1a2f116b-78f7-47b6-b96b-c0c74b5c5a35 +@bind reset_process_counter PlutoUI.CounterButton("Reset process!") + +# ╔═║ 8d6f46b6-e6cf-4094-a32a-f1b13dc005f6 +@bind send_counter PlutoUI.CounterButton("Send!") + +# ╔═║ 09f4b4b6-75a4-4727-9405-e4fc45c2cda7 +import Distributed + +# ╔═║ e5edaa4d-74ff-4f6e-a045-71fd5494dd79 +# @use_memo([spawned_process]) do +# try +# ChildProcesses.create_channel(spawned_process, quote +# Channel() do ch +# for i in 1:2000 +# put!(ch, i) +# end +# end +# end) do ch +# for i in 1:10 +# x = take!(ch) +# end +# end +# catch error +# @error "UHHH" error stacktrace(catch_backtrace()) +# (error, stacktrace(catch_backtrace())) +# end +# end + +# ╔═║ 5fb236c3-b67d-47ee-8644-84bd51e577b1 +# @task_result([spawned_process]) do +# ChildProcesses.call(spawned_process, quote +# 1 + 1 +# end) +# end + +# ╔═║ 8bd24b7b-4837-46b7-a6e9-b674630c2f56 +# @use_memo([spawned_process]) do +# for i in 1:10 +# ChildProcesses.call(spawned_process, quote +# 1 + 1 +# end) +# end +# end + +# ╔═║ e33a9b31-722e-425e-be5e-b33517bec8e3 +# @use_memo([spawned_process]) do +# BenchmarkTools.@benchmark ChildProcesses.call(spawned_process, quote +# 1 + 1 +# end) +# end + +# ╔═║ 33bcb6d2-5aea-4b7d-b5c4-e0dd2abc73f1 +1 + 1 + +# ╔═║ 2db3c415-e827-4ecb-a2c9-f407650c37ac +@bind g PlutoUI.Slider(1:100) + +# ╔═║ 4ac2d47a-30b5-4095-a873-258b003539cf +h = g + 2 + +# ╔═║ de602aaa-704c-45c4-8b7b-fc58e41236ce +begin + struct FakeProcess <: Base.AbstractPipe + in::IO + out::IO + end + eval(:(Base.process_running(::FakeProcess) = true)) + eval(:(Base.pipe_writer(process::FakeProcess) = process.in)) + eval(:(Base.pipe_reader(process::FakeProcess) = process.out)) +end + +# ╔═║ abf694f4-c5c4-4a4f-b30f-62e358149195 +# @task_result() do +# Base.readbytes!(PipeBuffer(), UInt8[], 10) +# end + +# ╔═║ f8bf6cc4-5ccd-4f0d-b7de-0f192bb3dfb1 +buffer = IOBuffer() + +# ╔═║ c96496a2-50bd-475c-ae02-f9b828080610 +@which write(buffer, UInt8[]) + +# ╔═║ 25a3fc9c-a96d-4077-a304-c59554e8f568 +@which unsafe_write(buffer, pointer(UInt8[]), sizeof(UInt8[])) + +# ╔═║ ec4a5558-28dd-47c1-b015-8799d9cb8800 +function Base.eof(buffer::IOBuffer) + while buffer.readable + if bytesavailable(buffer) !== 0 + return false + end + # eval(:(@info "OOPS")) + sleep(1) + yield() + end + return true +end + +# ╔═║ 281b4aab-307a-4d90-9dfb-f422b9567736 +process_output = let + error("Nope") + + my_stderr = @use_memo([reset_process_counter, ChildProcesses]) do + Pipe() + end + output, set_output = @use_state("", [my_stderr]) + + process = @use_memo([my_stderr]) do + ChildProcesses.create_child_process( + custom_stderr=my_stderr, + exeflags=["--color=yes", "--threads=4"], + ) + end + @use_effect([process]) do + return () -> begin + kill(process) + end + end + + # So we re-run the whole thing when the process exists + _, refresh_state = @use_state(nothing, [process]) + @background begin + if process_running(process) + wait(process) + refresh_state(nothing) + end + end + + @background begin + while process_running(process) && !eof(my_stderr) + new_output = String(readavailable(my_stderr)) + set_output((output) -> begin + output * new_output + end) + end + end + + pluto_process = @use_memo([process, my_stderr]) do + PlutoProcess( + process=process, + stderr=my_stderr, + stdin=process.process.in, + stdout=process.process.out, + ) + end + + PlutoUI.with_terminal(show_value=true) do + print(output) + pluto_process + end +end + +# ╔═║ 4be7e097-72a3-4590-bcfb-a7dacb78159c +spawned_process = process_output.value.process; + +# ╔═║ 49fdc8a3-0e1a-42f0-acc4-b823eec91d31 +md"---" + +# ╔═║ 95c5a5bc-db23-4ad3-8ae8-81bc8f0edfd4 +import BenchmarkTools + +# ╔═║ ced9d1e9-7075-4ff2-8ca2-6a349f2a69c4 +# let +# stream = PipeBuffer() +# BenchmarkTools.@benchmark begin +# ChildProcesses.send_message($stream, ChildProcesses.to_binary(Dict(:x => 1))) +# ChildProcesses.from_binary(ChildProcesses.read_message($stream)) +# end +# end + +# ╔═║ be18d157-6b55-4eaa-99fe-c398e992a9fa +md"### @task_result" + +# ╔═║ 12578b59-0161-4e72-afef-825166a62121 +struct Pending end + +# ╔═║ 34d90560-5a3e-4c7f-8126-35e1a6153aa1 +struct Result value end + +# ╔═║ 83540498-f317-4d8b-8dc6-80a93247b3b2 +struct Failure error end + +# ╔═║ 78803cff-2af8-4266-8fad-6911faf17910 +macro task_result(expr, deps=nothing) + quote + result, set_result = @use_state($(Pending)()) + + @background($(esc(deps))) do + try + result = $(esc(expr))() + set_result($(Result)(result)) + catch error + set_result($(Failure)(error)) + end + end + + result + end +end + +# ╔═║ ce384249-52c5-47d8-9c93-18837432b625 +let + parent_to_child = PipeBuffer() + child_to_parent = PipeBuffer() + child_process = @use_memo([ChildProcesses.ChildProcess, FakeProcess]) do + process = FakeProcess(child_to_parent, parent_to_child) + child_process = ChildProcesses.ChildProcess(process=process) + end + parent_process = @use_memo([ChildProcesses.ParentProcess]) do + ChildProcesses.ParentProcess( + parent_to_child=parent_to_child, + child_to_parent=child_to_parent, + ) + end + + @info "#1" + @background([ChildProcesses.start_from_child_loop, child_process]) do + ChildProcesses.start_from_child_loop(child_process) + end + @info "#2" + @background([ChildProcesses.listen_for_messages_from_parent, parent_process]) do + ChildProcesses.listen_for_messages_from_parent(parent_process) + end + @info "#3" + @task_result([ChildProcesses.call, child_process]) do + ChildProcesses.call(child_process, quote + 1 + 1 + end) + end +end + +# ╔═║ 00000000-0000-0000-0000-000000000001 +PLUTO_PROJECT_TOML_CONTENTS = """ +[deps] +BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" +PlutoUI = "7f904dfe-b85e-4ff6-b463-dae2292396a8" +Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[compat] +BenchmarkTools = "~1.2.0" +PlutoUI = "~0.7.16" +""" + +# ╔═║ 00000000-0000-0000-0000-000000000002 +PLUTO_MANIFEST_TOML_CONTENTS = """ +# This file is machine-generated - editing it directly is not advised + +[[Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[BenchmarkTools]] +deps = ["JSON", "Logging", "Printf", "Profile", "Statistics", "UUIDs"] +git-tree-sha1 = "61adeb0823084487000600ef8b1c00cc2474cd47" +uuid = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +version = "1.2.0" + +[[Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[Hyperscript]] +deps = ["Test"] +git-tree-sha1 = "8d511d5b81240fc8e6802386302675bdf47737b9" +uuid = "47d2ed2b-36de-50cf-bf87-49c2cf4b8b91" +version = "0.0.4" + +[[HypertextLiteral]] +git-tree-sha1 = "5efcf53d798efede8fee5b2c8b09284be359bf24" +uuid = "ac1192a8-f4b3-4bfe-ba22-af5b92cd3ab2" +version = "0.9.2" + +[[IOCapture]] +deps = ["Logging", "Random"] +git-tree-sha1 = "f7be53659ab06ddc986428d3a9dcc95f6fa6705a" +uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89" +version = "0.2.2" + +[[InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[JSON]] +deps = ["Dates", "Mmap", "Parsers", "Unicode"] +git-tree-sha1 = "8076680b162ada2a031f707ac7b4953e30667a37" +uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +version = "0.21.2" + +[[Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[LinearAlgebra]] +deps = ["Libdl"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + +[[Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" + +[[Parsers]] +deps = ["Dates"] +git-tree-sha1 = "d911b6a12ba974dabe2291c6d450094a7226b372" +uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" +version = "2.1.1" + +[[PlutoUI]] +deps = ["Base64", "Dates", "Hyperscript", "HypertextLiteral", "IOCapture", "InteractiveUtils", "JSON", "Logging", "Markdown", "Random", "Reexport", "UUIDs"] +git-tree-sha1 = "4c8a7d080daca18545c56f1cac28710c362478f3" +uuid = "7f904dfe-b85e-4ff6-b463-dae2292396a8" +version = "0.7.16" + +[[Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[Profile]] +deps = ["Printf"] +uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" + +[[Random]] +deps = ["Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[Reexport]] +git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" +uuid = "189a3867-3050-52da-a836-e630ba90ab69" +version = "1.2.2" + +[[SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" + +[[Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[SparseArrays]] +deps = ["LinearAlgebra", "Random"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + +[[Statistics]] +deps = ["LinearAlgebra", "SparseArrays"] +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" + +[[Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +""" + +# ╔═║ Cell order: +# ╠═7e613bd2-616a-4687-8af5-a22c7a747d97 +# ╠═d30e1e1b-fa6f-4fbc-bccb-1fcfc7b829df +# ╠═a901d255-c18c-45ed-9827-afd79246613c +# ╠═f7d14367-27d7-41a5-9f6a-79cf5e721a7d +# ╠═f9675ee0-e728-420b-81bd-22e57583c587 +# ╠═c9a9a089-037b-4fa8-91b2-980a318faee7 +# β•Ÿβ”€ca96e0d5-0904-4ae5-89d0-c1a9187710a1 +# β•Ÿβ”€b7734627-ba2a-48d8-8fd8-a5c94716da20 +# β•Ÿβ”€1a2f116b-78f7-47b6-b96b-c0c74b5c5a35 +# β•Ÿβ”€8d6f46b6-e6cf-4094-a32a-f1b13dc005f6 +# ╠═09f4b4b6-75a4-4727-9405-e4fc45c2cda7 +# ╠═281b4aab-307a-4d90-9dfb-f422b9567736 +# ╠═e5edaa4d-74ff-4f6e-a045-71fd5494dd79 +# ╠═5fb236c3-b67d-47ee-8644-84bd51e577b1 +# ╠═8bd24b7b-4837-46b7-a6e9-b674630c2f56 +# ╠═e33a9b31-722e-425e-be5e-b33517bec8e3 +# ╠═33bcb6d2-5aea-4b7d-b5c4-e0dd2abc73f1 +# ╠═2db3c415-e827-4ecb-a2c9-f407650c37ac +# ╠═4ac2d47a-30b5-4095-a873-258b003539cf +# ╠═de602aaa-704c-45c4-8b7b-fc58e41236ce +# ╠═abf694f4-c5c4-4a4f-b30f-62e358149195 +# ╠═f8bf6cc4-5ccd-4f0d-b7de-0f192bb3dfb1 +# ╠═c96496a2-50bd-475c-ae02-f9b828080610 +# ╠═25a3fc9c-a96d-4077-a304-c59554e8f568 +# ╠═ec4a5558-28dd-47c1-b015-8799d9cb8800 +# ╠═ce384249-52c5-47d8-9c93-18837432b625 +# ╠═4be7e097-72a3-4590-bcfb-a7dacb78159c +# ╠═49fdc8a3-0e1a-42f0-acc4-b823eec91d31 +# ╠═95c5a5bc-db23-4ad3-8ae8-81bc8f0edfd4 +# ╠═ced9d1e9-7075-4ff2-8ca2-6a349f2a69c4 +# β•Ÿβ”€be18d157-6b55-4eaa-99fe-c398e992a9fa +# ╠═12578b59-0161-4e72-afef-825166a62121 +# ╠═34d90560-5a3e-4c7f-8126-35e1a6153aa1 +# ╠═83540498-f317-4d8b-8dc6-80a93247b3b2 +# ╠═78803cff-2af8-4266-8fad-6911faf17910 +# β•Ÿβ”€00000000-0000-0000-0000-000000000001 +# β•Ÿβ”€00000000-0000-0000-0000-000000000002 diff --git a/src/evaluation/Run.jl b/src/evaluation/Run.jl index 25771f6621..992586d5f9 100644 --- a/src/evaluation/Run.jl +++ b/src/evaluation/Run.jl @@ -385,7 +385,7 @@ function update_save_run!(session::ServerSession, notebook::Notebook, cells::Arr session.options.server.disable_writing_notebook_files || (save && save_notebook(notebook)) # _assume `prerender_text == false` if you want to skip some details_ - to_run_online = if !prerender_text + to_run_online = if !prerender_text || true cells else # this code block will run cells that only contain text offline, i.e. on the server process, before doing anything else diff --git a/src/evaluation/WorkspaceManager.jl b/src/evaluation/WorkspaceManager.jl index b789893cae..e04ce330ba 100644 --- a/src/evaluation/WorkspaceManager.jl +++ b/src/evaluation/WorkspaceManager.jl @@ -6,13 +6,13 @@ import ..Pluto.PkgCompat import ..Configuration: CompilerOptions, _merge_notebook_compiler_options, _convert_to_flags import ..Pluto.ExpressionExplorer: FunctionName import ..PlutoRunner -import Distributed +import ..ChildProcesses "Contains the Julia process (in the sense of `Distributed.addprocs`) to evaluate code in. Each notebook gets at most one `Workspace` at any time, but it can also have no `Workspace` (it cannot `eval` code in this case)." Base.@kwdef mutable struct Workspace - pid::Integer + process::Any discarded::Bool=false - log_channel::Distributed.RemoteChannel + log_channel::ChildProcesses.ChildChannel module_name::Symbol dowork_token::Token=Token() nbpkg_was_active::Bool=false @@ -23,9 +23,9 @@ end "These expressions get evaluated whenever a new `Workspace` process is created." const process_preamble = [ :(ccall(:jl_exit_on_sigint, Cvoid, (Cint,), 0)), - :(include($(project_relative_path("src", "runner", "Loader.jl")))), :(ENV["GKSwstype"] = "nul"), :(ENV["JULIA_REVISE_WORKER_ONLY"] = "1"), + :(include($(project_relative_path("src", "runner", "Loader.jl")))), ] const workspaces = Dict{UUID,Promise{Workspace}}() @@ -42,42 +42,54 @@ function make_workspace((session, notebook)::SN; force_offline::Bool=false)::Wor session.options.evaluation.workspace_use_distributed end - pid = if use_distributed + process = if use_distributed create_workspaceprocess(;compiler_options=_merge_notebook_compiler_options(notebook, session.options.compiler)) else - pid = Distributed.myid() - if !(isdefined(Main, :PlutoRunner) && Main.PlutoRunner isa Module) - # we make PlutoRunner available in Main, right now it's only defined inside this Pluto module. - @eval Main begin - PlutoRunner = $(PlutoRunner) - end - end - pid + error("mehh") + # pid = Distributed.myid() + # if !(isdefined(Main, :PlutoRunner) && Main.PlutoRunner isa Module) + # # we make PlutoRunner available in Main, right now it's only defined inside this Pluto module. + # @eval Main begin + # PlutoRunner = $(PlutoRunner) + # end + # end + # pid end - Distributed.remotecall_eval(Main, [pid], :(PlutoRunner.notebook_id[] = $(notebook.notebook_id))) - log_channel = Core.eval(Main, quote - $(Distributed).RemoteChannel(() -> eval(:(Main.PlutoRunner.log_channel)), $pid) - end) - run_channel = Core.eval(Main, quote - $(Distributed).RemoteChannel(() -> eval(:(Main.PlutoRunner.run_channel)), $pid) + ChildProcesses.call_without_fetch(process, quote + PlutoRunner.notebook_id[] = $(notebook.notebook_id) end) - module_name = create_emptyworkspacemodule(pid) + # Distributed.remotecall_eval(Main, [pid], :(PlutoRunner.notebook_id[] = $(notebook.notebook_id))) + log_channel = ChildProcesses.create_channel(process, :(Main.PlutoRunner.log_channel)) + # log_channel = Core.eval(Main, quote + # $(Distributed).RemoteChannel(() -> eval(:(Main.PlutoRunner.log_channel)), $pid) + # end) + run_channel = ChildProcesses.create_channel(process, :(Main.PlutoRunner.run_channel)) + + module_name = create_emptyworkspacemodule(process) - original_LOAD_PATH, original_ACTIVE_PROJECT = Distributed.remotecall_eval(Main, pid, :(Base.LOAD_PATH, Base.ACTIVE_PROJECT[])) + original_LOAD_PATH, original_ACTIVE_PROJECT = ChildProcesses.call(process, :(Base.LOAD_PATH, Base.ACTIVE_PROJECT[])) + @warn "#1" + workspace = Workspace(; - pid=pid, + process=process, log_channel=log_channel, module_name=module_name, original_LOAD_PATH=original_LOAD_PATH, original_ACTIVE_PROJECT=original_ACTIVE_PROJECT, ) + @warn "#2" + @async start_relaying_logs((session, notebook), log_channel) + @warn "#3" @async start_relaying_self_updates((session, notebook), run_channel) + @warn "#4" cd_workspace(workspace, notebook.path) + @warn "#5" use_nbpkg_environment((session, notebook), workspace) + @warn "#6" force_offline || (notebook.process_status = ProcessStatus.ready) return workspace @@ -95,11 +107,11 @@ function use_nbpkg_environment((session, notebook)::SN, workspace=nothing) end workspace.nbpkg_was_active = enabled - if workspace.pid != Distributed.myid() + if !(workspace.process isa ChildProcesses.LocalSandbox) new_LP = enabled ? ["@", "@stdlib"] : workspace.original_LOAD_PATH new_AP = enabled ? PkgCompat.env_dir(notebook.nbpkg_ctx) : workspace.original_ACTIVE_PROJECT - Distributed.remotecall_eval(Main, [workspace.pid], quote + ChildProcesses.call(workspace.process, quote copy!(LOAD_PATH, $(new_LP)) Base.ACTIVE_PROJECT[] = $(new_AP) end) @@ -108,7 +120,7 @@ function use_nbpkg_environment((session, notebook)::SN, workspace=nothing) end end -function start_relaying_self_updates((session, notebook)::SN, run_channel::Distributed.RemoteChannel) +function start_relaying_self_updates((session, notebook)::SN, run_channel::ChildProcesses.ChildChannel) while true try next_run_uuid = take!(run_channel) @@ -124,7 +136,7 @@ function start_relaying_self_updates((session, notebook)::SN, run_channel::Distr end end -function start_relaying_logs((session, notebook)::SN, log_channel::Distributed.RemoteChannel) +function start_relaying_logs((session, notebook)::SN, log_channel::ChildProcesses.ChildChannel) while true try next_log = take!(log_channel) @@ -149,13 +161,13 @@ end function bump_workspace_module(session_notebook::SN) workspace = get_workspace(session_notebook) old_name = workspace.module_name - new_name = workspace.module_name = create_emptyworkspacemodule(workspace.pid) + new_name = workspace.module_name = create_emptyworkspacemodule(workspace.process) old_name, new_name end -function create_emptyworkspacemodule(pid::Integer)::Symbol - Distributed.remotecall_eval(Main, pid, :(PlutoRunner.increment_current_module())) +function create_emptyworkspacemodule(process)::Symbol + ChildProcesses.call(process, :(PlutoRunner.increment_current_module())) end const Distributed_expr = :( @@ -165,26 +177,27 @@ const Distributed_expr = :( # NOTE: this function only start a worker process using given # compiler options, it does not resolve paths for notebooks # compiler configurations passed to it should be resolved before this -function create_workspaceprocess(;compiler_options=CompilerOptions())::Integer +function create_workspaceprocess(;compiler_options=CompilerOptions()) # run on proc 1 in case Pluto is being used inside a notebook process # Workaround for "only process 1 can add/remove workers" - pid = Distributed.remotecall_eval(Main, 1, quote - $(Distributed_expr).addprocs(1; exeflags=$(_convert_to_flags(compiler_options))) |> first - end) + + exeflags = _convert_to_flags(compiler_options) + process = ChildProcesses.create_child_process(exeflags=exeflags) for expr in process_preamble - Distributed.remotecall_eval(Main, [pid], expr) + ChildProcesses.call_without_fetch(process, expr) end # so that we NEVER break the workspace with an interrupt πŸ€• - @async Distributed.remotecall_eval(Main, [pid], - :(while true + @async ChildProcesses.call_without_fetch(process, quote + while true try wait() catch end - end)) + end + end) - pid + process end "Return the `Workspace` of `notebook`; will be created if none exists yet." @@ -202,15 +215,13 @@ function unmake_workspace(session_notebook::Union{SN,Workspace}; async=false) workspace = get_workspace(session_notebook) workspace.discarded = true - if workspace.pid != Distributed.myid() - filter!(p -> fetch(p.second).pid != workspace.pid, workspaces) + if !(workspace.process isa ChildProcesses.LocalSandbox) + filter!(p -> fetch(p.second).process != workspace.process, workspaces) t = @async begin interrupt_workspace(workspace; verbose=false) # run on proc 1 in case Pluto is being used inside a notebook process # Workaround for "only process 1 can add/remove workers" - Distributed.remotecall_eval(Main, 1, quote - $(Distributed_expr).rmprocs($(workspace.pid)) - end) + close(workspace.process) end async || wait(t) end @@ -232,8 +243,8 @@ end function distributed_exception_result(exs::CompositeException, workspace::Workspace) ex = exs.exceptions |> first - if ex isa Distributed.RemoteException && - ex.pid == workspace.pid && + if ex isa ChildProcesses.ChildProcessException && + ex.process == workspace.process && ex.captured.ex isa InterruptException ( @@ -244,7 +255,7 @@ function distributed_exception_result(exs::CompositeException, workspace::Worksp runtime=nothing, published_objects=Dict{String,Any}(), ) - elseif ex isa Distributed.ProcessExitedException + elseif ex isa ChildProcesses.ProcessExitedException ( output_formatted=PlutoRunner.format_output(CapturedException(exs, [])), errored=true, @@ -278,30 +289,31 @@ function eval_format_fetch_in_workspace( function_wrapped_info::Union{Nothing,Tuple}=nothing, forced_expr_id::Union{PlutoRunner.ObjectID,Nothing}=nothing, )::NamedTuple{(:output_formatted, :errored, :interrupted, :process_exited, :runtime, :published_objects),Tuple{PlutoRunner.MimedOutput,Bool,Bool,Bool,Union{UInt64,Nothing},Dict{String,Any}}} - workspace = get_workspace(session_notebook) # if multiple notebooks run on the same process, then we need to `cd` between the different notebook paths if session_notebook isa Tuple - if workspace.pid == Distributed.myid() + if workspace.process isa ChildProcesses.LocalSandbox cd_workspace(workspace, session_notebook[2].path) end use_nbpkg_environment(session_notebook, workspace) end - + # run the code πŸƒβ€β™€οΈ # a try block (on this process) to catch an InterruptException take!(workspace.dowork_token) early_result = try # we use [pid] instead of pid to prevent fetching output - Distributed.remotecall_eval(Main, [workspace.pid], :(PlutoRunner.run_expression( - getfield(Main, $(QuoteNode(workspace.module_name))), - $(QuoteNode(expr)), - $cell_id, - $function_wrapped_info, - $forced_expr_id, - ))) + ChildProcesses.call_without_fetch(workspace.process, quote + PlutoRunner.run_expression( + getfield(Main, $(QuoteNode(workspace.module_name))), + $(QuoteNode(expr)), + $cell_id, + $function_wrapped_info, + $forced_expr_id, + ) + end) put!(workspace.dowork_token) nothing catch exs @@ -319,7 +331,9 @@ end function eval_in_workspace(session_notebook::Union{SN,Workspace}, expr) workspace = get_workspace(session_notebook) - Distributed.remotecall_eval(Main, [workspace.pid], :(Core.eval($(workspace.module_name), $(expr |> QuoteNode)))) + ChildProcesses.call_without_fetch(workspace.process, quote + Core.eval($(workspace.module_name), $(expr |> QuoteNode)) + end) nothing end @@ -329,10 +343,12 @@ function format_fetch_in_workspace(session_notebook::Union{SN,Workspace}, cell_i # instead of fetching the output value (which might not make sense in our context, since the user can define structs, types, functions, etc), we format the cell output on the worker, and fetch the formatted output. withtoken(workspace.dowork_token) do try - Distributed.remotecall_eval(Main, workspace.pid, :(PlutoRunner.formatted_result_of( - $cell_id, $ends_with_semicolon, $showmore_id, - getfield(Main, $(QuoteNode(workspace.module_name))), - ))) + ChildProcesses.call(workspace.process, quote + PlutoRunner.formatted_result_of( + $cell_id, $ends_with_semicolon, $showmore_id, + getfield(Main, $(QuoteNode(workspace.module_name))), + ) + end) catch ex distributed_exception_result(CompositeException([ex]), workspace) end @@ -347,7 +363,7 @@ function collect_soft_definitions(session_notebook::SN, modules::Set{Expr}) PlutoRunner.collect_soft_definitions($module_name, $modules) end - Distributed.remotecall_eval(Main, workspace.pid, ex) + ChildProcesses.call(workspace.process, ex) end @@ -355,7 +371,7 @@ function macroexpand_in_workspace(session_notebook::Union{SN,Workspace}, macroca workspace = get_workspace(session_notebook) module_name = module_name === nothing ? workspace.module_name : module_name - Distributed.remotecall_eval(Main, workspace.pid, quote + ChildProcesses.call(workspace.process, quote try (true, PlutoRunner.try_macroexpand($(module_name), $(cell_uuid), $(macrocall |> QuoteNode))) catch e @@ -368,13 +384,15 @@ end function eval_fetch_in_workspace(session_notebook::Union{SN,Workspace}, expr) workspace = get_workspace(session_notebook) - Distributed.remotecall_eval(Main, workspace.pid, :(Core.eval($(workspace.module_name), $(expr |> QuoteNode)))) + ChildProcesses.call(workspace.process, :(Core.eval($(workspace.module_name), $(expr |> QuoteNode)))) end function do_reimports(session_notebook::Union{SN,Workspace}, module_imports_to_move::Set{Expr}) workspace = get_workspace(session_notebook) workspace_name = workspace.module_name - Distributed.remotecall_eval(Main, [workspace.pid], :(PlutoRunner.do_reimports($(workspace_name), $module_imports_to_move))) + ChildProcesses.call_without_fetch(workspace.process, quote + PlutoRunner.do_reimports($(workspace_name), $module_imports_to_move) + end) end "Move variables to a new module. A given set of variables to be 'deleted' will not be moved to the new module, making them unavailable. " @@ -382,7 +400,9 @@ function move_vars(session_notebook::Union{SN,Workspace}, old_workspace_name::Sy workspace = get_workspace(session_notebook) new_workspace_name = something(new_workspace_name, workspace.module_name) - Distributed.remotecall_eval(Main, [workspace.pid], :(PlutoRunner.move_vars($(old_workspace_name |> QuoteNode), $(new_workspace_name |> QuoteNode), $to_delete, $methods_to_delete, $module_imports_to_move))) + ChildProcesses.call_without_fetch(workspace.process, quote + PlutoRunner.move_vars($(old_workspace_name |> QuoteNode), $(new_workspace_name |> QuoteNode), $to_delete, $methods_to_delete, $module_imports_to_move) + end) end move_vars(session_notebook::Union{SN,Workspace}, to_delete::Set{Symbol}, methods_to_delete::Set{Tuple{UUID,FunctionName}}, module_imports_to_move::Set{Expr}; kwargs...) = @@ -409,6 +429,8 @@ end function interrupt_workspace(session_notebook::Union{SN,Workspace}; verbose=true)::Bool workspace = get_workspace(session_notebook) + verbose=true + if poll(() -> isready(workspace.dowork_token), 2.0, 5/100) verbose && println("Cell finished, other cells cancelled!") return true @@ -420,7 +442,7 @@ function interrupt_workspace(session_notebook::Union{SN,Workspace}; verbose=true https://docs.microsoft.com/en-us/windows/wsl" return false end - if workspace.pid == Distributed.myid() + if workspace.process isa ChildProcesses.LocalSandbox verbose && @warn """Cells in this workspace can't be stopped, because it is not running in a separate workspace. Use `ENV["PLUTO_WORKSPACE_USE_DISTRIBUTED"]` to control whether future workspaces are generated in a separate process.""" return false end @@ -435,7 +457,7 @@ function interrupt_workspace(session_notebook::Union{SN,Workspace}; verbose=true # TODO: listen for the final words of the remote process on stdout/stderr: "Force throwing a SIGINT" try verbose && @info "Sending interrupt to process $(workspace.pid)" - Distributed.interrupt(workspace.pid) + kill(workspace.process, Base.SIGINT) if poll(() -> isready(workspace.dowork_token), 5.0, 5/100) verbose && println("Cell interrupted!") @@ -446,7 +468,7 @@ function interrupt_workspace(session_notebook::Union{SN,Workspace}; verbose=true while !isready(workspace.dowork_token) for _ in 1:5 verbose && print(" πŸ”₯ ") - Distributed.interrupt(workspace.pid) + kill(workspace.process, Base.SIGINT) sleep(0.18) if isready(workspace.dowork_token) break diff --git a/src/webserver/REPLTools.jl b/src/webserver/REPLTools.jl index b4b5fd9b35..171c6b0898 100644 --- a/src/webserver/REPLTools.jl +++ b/src/webserver/REPLTools.jl @@ -1,9 +1,9 @@ import FuzzyCompletions: complete_path, completion_text, score -import Distributed import .PkgCompat: package_completions using Markdown import REPL + ### # RESPONSES FOR AUTOCOMPLETE & DOCS ### @@ -79,10 +79,10 @@ responses[:complete] = function response_complete(πŸ™‹::ClientRequest) if will_run_code(πŸ™‹.notebook) && isready(workspace.dowork_token) # we don't use eval_format_fetch_in_workspace because we don't want the output to be string-formatted. # This works in this particular case, because the return object, a `Completion`, exists in this scope too. - Distributed.remotecall_eval(Main, workspace.pid, :(PlutoRunner.completion_fetcher( + ChildProcesses.call(workspace.process, :(PlutoRunner.completion_fetcher( $query, $pos, getfield(Main, $(QuoteNode(workspace.module_name))), - ))) + ))) else # We can at least autocomplete general julia things: PlutoRunner.completion_fetcher(query, pos, Main) @@ -121,7 +121,7 @@ responses[:docs] = function response_docs(πŸ™‹::ClientRequest) workspace = WorkspaceManager.get_workspace((πŸ™‹.session, πŸ™‹.notebook)) if will_run_code(πŸ™‹.notebook) && isready(workspace.dowork_token) - Distributed.remotecall_eval(Main, workspace.pid, :(PlutoRunner.doc_fetcher( + ChildProcesses.call(workspace.process, :(PlutoRunner.doc_fetcher( $query, getfield(Main, $(QuoteNode(workspace.module_name))), ))) From 8be0bb2c160cd1640e5abc4136112fa1d761cb9e Mon Sep 17 00:00:00 2001 From: Michiel Dral Date: Tue, 25 Jan 2022 19:37:45 +0100 Subject: [PATCH 24/28] Trying --- src/evaluation/ChildProcessesNotebook.jl | 371 ++++++++++++----------- src/evaluation/ChildProcessesTest.jl | 350 +++++++++++---------- src/evaluation/WorkspaceManager.jl | 25 +- 3 files changed, 412 insertions(+), 334 deletions(-) diff --git a/src/evaluation/ChildProcessesNotebook.jl b/src/evaluation/ChildProcessesNotebook.jl index ee920e904b..cf177653d5 100644 --- a/src/evaluation/ChildProcessesNotebook.jl +++ b/src/evaluation/ChildProcessesNotebook.jl @@ -1,85 +1,23 @@ ### A Pluto.jl notebook ### -# v0.17.0 +# v0.17.7 using Markdown using InteractiveUtils +# ╔═║ 4c4b77dc-8545-4922-9897-110fa67c99f4 +md"# ChildProcesses" + # ╔═║ 6925ffbb-fd9e-402c-a483-c78f28f892a5 import Serialization # ╔═║ 8c97d5cb-e346-4b94-b2a1-4fb5ff093bd5 import UUIDs: UUID, uuid4 -# ╔═║ a6e50946-dd3d-4738-adc1-26534e184776 -md""" -## Binary message frame thing - -Functions to send and read binary messages over a stable connection, also functions to serialize to a bytearray. - -Pretty simple right now, might want to add cooler stuff later. -""" - -# ╔═║ 131a123b-e319-4939-90bf-7fb035ab2e75 -MessageLength = Int - -# ╔═║ e393ff80-3995-11ec-1217-b3642a509067 -function read_message(stream) - message_length_buffer = Vector{UInt8}(undef, 8) - bytesread = readbytes!(stream, message_length_buffer, 8) - how_long_will_the_message_be = reinterpret(MessageLength, message_length_buffer)[1] - - message_buffer = Vector{UInt8}(undef, how_long_will_the_message_be) - _bytesread = readbytes!(stream, message_buffer, how_long_will_the_message_be) - - message_buffer -end - -# ╔═║ 52495855-a52d-4edf-9c7c-811a5060e641 -function to_binary(message) - io = PipeBuffer() - serialized = Serialization.serialize(io, message) - read(io) -end - -# ╔═║ 7da416d1-5dfb-4510-9d32-62c1464a83d4 -function from_binary(message) - Serialization.deserialize(IOBuffer(message)) -end - # ╔═║ 87612815-7981-4a69-a65b-4465f48c9fd9 struct ReadMessages io::IO end -# ╔═║ 009ad714-b759-420a-b49a-6caed7ee3faf -function Base.iterate(message_reader::ReadMessages, state=nothing) - @warn "Iterating!" - if eof(message_reader.io) - @warn "Pfff" - return nothing - end - @warn "Nice" - - message = from_binary(read_message(message_reader.io)) - (message, nothing) -end - -# ╔═║ faf3c68e-d0fb-4dd9-ac1b-1ff8d922134b -function send_message(stream, bytes) - # bytes = Vector{UInt8}(message) - message_length = convert(MessageLength, length(bytes)) - # @info "message_length" message_length bytes - how_long_will_the_message_be = reinterpret(UInt8, [message_length]) - - @warn "Writing to stream!!!" - _1 = write(stream, how_long_will_the_message_be) - _2 = write(stream, bytes) - @warn "WROTE TO STREAM" - - _1 + _2 - @info "Write" bytesavailable(stream.in) _1 _2 -end - # ╔═║ de680a32-e053-4655-a1cd-111d81a037e6 md""" ## Child process side @@ -138,6 +76,20 @@ Base.@kwdef struct ProcessExitedException <: Exception process::ChildProcess end +# ╔═║ 2a538f97-a9d1-4dcf-a098-5b1e5a8df3ae +function Base.showerror(io::IO, error::ProcessExitedException) + print(io, "ChildProcessExitedException: Child process has exited") +end + +# ╔═║ 9ba8ee0c-c80c-41d9-8739-11e6ef5d3c15 +function Base.showerror(io::IO, error::ChildProcessException) + print(io, "Child process has an error:") + Base.showerror(io, error.captured) +end + +# ╔═║ 368ccd31-1681-4a4a-a103-2b9afc0813ee +Base.kill(process::ChildProcess, signal::Int) = Base.kill(process.process, signal) + # ╔═║ af4f0720-f7f7-4d8a-bd8c-8cf5abaf10a0 Base.kill(process::ChildProcess) = Base.kill(process.process) @@ -147,33 +99,12 @@ Base.process_running(p::ChildProcess) = Base.process_running(p.process) # ╔═║ e787d8bd-499d-4a41-8913-62b3d9346748 Base.wait(process::ChildProcess) = Base.wait(process.process) -# ╔═║ 3a62b419-eca2-4de1-89a4-fc9ad6f68372 -struct SingleTakeChannel end - -# ╔═║ 61410bd1-8456-4608-92d9-3c863f65b89c -function Base.take!(::SingleTakeChannel) - nothing -end - -# ╔═║ 409707ff-b1ea-453d-b933-0a4b1e5f44c8 -function Base.iterate(::SingleTakeChannel, _=nothing) - (nothing, nothing) -end - -# ╔═║ 0feb758e-38d4-48c0-a5e9-9d1129b1b1b2 -function Base.isopen(::SingleTakeChannel) - true -end - # ╔═║ abe897ff-6e86-40eb-a1a6-2918b6c3c5a7 struct LocalSandbox end # ╔═║ 934d18d4-936e-4ee0-aa6e-86aa6f66774c juliapath() = joinpath(Sys.BINDIR::String, Base.julia_exename()) -# ╔═║ b48f36d1-e3d2-408e-a705-93e609f2fd04 -fieldnames(Base.Process) - # ╔═║ 884e103a-8925-477d-a264-f15d02a49aa9 md""" ## ChildChannel @@ -226,6 +157,97 @@ Base.@kwdef struct Envelope message::Message end +# ╔═║ 1f868b3c-df9a-4d4d-a6a9-9a196298a3af +md"---" + +# ╔═║ 77df6f7b-ed8e-442a-9489-873fc392937c +md""" +## SingleTakeChannel() + +Simple channel that will yield "nothing" and nothing else. +""" + +# ╔═║ 3a62b419-eca2-4de1-89a4-fc9ad6f68372 +Base.@kwdef mutable struct SingleTakeChannel <: AbstractChannel{Nothing} + did_finish=false +end + +# ╔═║ 61410bd1-8456-4608-92d9-3c863f65b89c +function Base.take!(channel::SingleTakeChannel) + if channel.did_finish + throw("SingleTakeChannel re-used") + end + channel.did_finish = true + nothing +end + +# ╔═║ 409707ff-b1ea-453d-b933-0a4b1e5f44c8 +function Base.iterate(channel::SingleTakeChannel, _=nothing) + if channel.did_finish + return nothing + else + channel.did_finish = true + (nothing, nothing) + end +end + +# ╔═║ 0feb758e-38d4-48c0-a5e9-9d1129b1b1b2 +function Base.isopen(channel::SingleTakeChannel) + !channel.did_finish +end + +# ╔═║ b0bcdbdf-a043-4d25-8582-ed173006e040 +function Base.close(single_channel::SingleTakeChannel) + single_channel.did_finish = true +end + +# ╔═║ a6e50946-dd3d-4738-adc1-26534e184776 +md""" +## Binary message frame thing + +Functions to send and read binary messages over a stable connection, also functions to serialize to a bytearray. + +Pretty simple right now, might want to add cooler stuff later. +""" + +# ╔═║ 131a123b-e319-4939-90bf-7fb035ab2e75 +MessageLength = Int + +# ╔═║ e393ff80-3995-11ec-1217-b3642a509067 +function read_message(stream) + message_length_buffer = Vector{UInt8}(undef, 8) + bytesread = readbytes!(stream, message_length_buffer, 8) + how_long_will_the_message_be = reinterpret(MessageLength, message_length_buffer)[1] + + message_buffer = Vector{UInt8}(undef, how_long_will_the_message_be) + _bytesread = readbytes!(stream, message_buffer, how_long_will_the_message_be) + + message_buffer +end + +# ╔═║ faf3c68e-d0fb-4dd9-ac1b-1ff8d922134b +function send_message(stream, bytes) + # bytes = Vector{UInt8}(message) + message_length = convert(MessageLength, length(bytes)) + # @info "message_length" message_length bytes + how_long_will_the_message_be = reinterpret(UInt8, [message_length]) + + # @warn "Writing to stream!!!" + _1 = write(stream, how_long_will_the_message_be) + _2 = write(stream, bytes) + # @warn "WROTE TO STREAM" + + _1 + _2 + # @info "Write" _1 _2 +end + +# ╔═║ 52495855-a52d-4edf-9c7c-811a5060e641 +function to_binary(message) + io = PipeBuffer() + serialized = Serialization.serialize(io, message) + read(io) +end + # ╔═║ 590a7882-3d69-48b0-bb1b-f476c7f8a885 function respond_to_parent(; to::ParentProcess, about::UUID, with::Message) binary_message = to_binary(Envelope( @@ -234,7 +256,6 @@ function respond_to_parent(; to::ParentProcess, about::UUID, with::Message) )) lock(to.lock) try - # @info "Message out" about message=typeof(with) send_message(to.child_to_parent, binary_message) finally unlock(to.lock) @@ -250,7 +271,6 @@ function send_message_without_response(process::ChildProcess, envelope::Envelope lock(process.lock) try - @info "Message out" message=typeof(envelope.message) send_message(process.process, to_binary(envelope)) return envelope finally @@ -273,33 +293,31 @@ function Base.take!(channel::ChildChannel) take!(channel.local_channel) end -# ╔═║ d3ce3ecf-c1e4-4484-ac38-b332a4b13034 -begin - import Base: close - close(process::ChildProcess) = close(process.process) - function close(channel::ChildChannel) - if isopen(channel.local_channel) - try - close(channel.local_channel) - if process_running(channel.process.process) - send_message_without_response(channel.process, Envelope( - channel_id=channel.id, - message=ChannelCloseMessage() - )) - end - catch e - @error "Problem with closing?" e stack=stacktrace(catch_backtrace()) - nothing +# ╔═║ ed6c16ed-bf4a-40ce-9c14-a72fd77e24a1 +function Base.close(channel::ChildChannel) + if isopen(channel.local_channel) + try + close(channel.local_channel) + if process_running(channel.process.process) + send_message_without_response(channel.process, Envelope( + channel_id=channel.id, + message=ChannelCloseMessage() + )) end + catch e + @error "Problem with closing?" e stack=stacktrace(catch_backtrace()) + nothing end - delete!(channel.process.channels, channel.id) end - function close(::SingleTakeChannel) end + delete!(channel.process.channels, channel.id) end +# ╔═║ 4289b4ba-45f2-4be7-b983-68f7d97510fa +Base.close(process::ChildProcess) = Base.close(process.process) + # ╔═║ 6a7aa0ce-0ae3-4e7d-a93a-bfdebe406220 begin - function handle_child_message(; + function handle_child_message( message, process::ChildProcess, input_channel::AbstractChannel, @@ -308,7 +326,7 @@ begin @warn "Unknown message type" message end - function handle_child_message(; + function handle_child_message( message::ChannelPushMessage, process::ChildProcess, input_channel::AbstractChannel, @@ -317,7 +335,7 @@ begin put!(output_channel, message.value) end - function handle_child_message(; + function handle_child_message( message::ChannelCloseMessage, process::ChildProcess, input_channel::AbstractChannel, @@ -326,7 +344,7 @@ begin close(output_channel) end - function handle_child_message(; + function handle_child_message( message::ErrorMessage, process::ChildProcess, input_channel::AbstractChannel, @@ -350,20 +368,17 @@ function create_channel(process::ChildProcess, expr) response_channel = Channel{Message}() process.channels[channel_id] = response_channel - @info "CREATING CHANNEL" actual_values_channes = Channel(Inf) do ch try for message in response_channel - @info "MESSAGE FROM CHILD CHANNEL" channel_id message handle_child_message( - message=message, - process=process, - input_channel=response_channel, - output_channel=ch, + message, + process, + response_channel, + ch, ) end catch e - @error "ERROR???" e close(ch, e) finally close(ch) @@ -396,7 +411,6 @@ function call(process::ChildProcess, expr) put!(ch, $expr) end end) do channel - @info "Taking" take!(channel) end end @@ -412,21 +426,19 @@ function call_without_fetch(process::ChildProcess, expr) end # ╔═║ 54a03cba-6d00-4632-8bd6-c60753c15ae6 -function start_from_child_loop(child_process) +function listen_for_messages_from_child(child_process) try for result in ReadMessages(child_process.process) if !(result isa Envelope && result.channel_id !== nothing) throw("Huh") end - @warn "MESSAGE FROM CHILD" result.message if haskey(child_process.channels, result.channel_id) message = result.message channel = child_process.channels[result.channel_id] put!(channel, message) else - @error "No channel to push to" throw("No channel to push to") end end @@ -441,6 +453,15 @@ function start_from_child_loop(child_process) end # ╔═║ d3fe8144-6ba8-4dd9-b0e3-941a96422267 +""" + create_child_process(; custom_stderr=stderr, exeflags=["--history-file=no"]) + +Spawn a child process that you'll be able to `call()` on and communicate with over +`RemoteChannels`. + +It spawns a process using `open` with some ad-hoc julia code that also loads the `ChildProcesses` module. It then spawns a loop for listening to the spawned process. +The returned `ChildProcess` is basically a bunch of event listeners (in the form of channels) that +""" function create_child_process(; custom_stderr=stderr, exeflags=["--history-file=no"]) this_file = split(@__FILE__(), "#")[1] this_module = string(nameof(@__MODULE__)) @@ -450,13 +471,10 @@ function create_child_process(; custom_stderr=stderr, exeflags=["--history-file= var"$this_module" = @eval Main module var"$this_module" include("$this_file") end - - @info "Booted up!" - + const parent_process = var"$this_module".setup_parent_process() try - @info "Startin..." var"$this_module".listen_for_messages_from_parent(parent_process) @info "Done?" catch error @@ -473,11 +491,26 @@ function create_child_process(; custom_stderr=stderr, exeflags=["--history-file= child_process = ChildProcess(process=process) schedule(Task() do - start_from_child_loop(child_process) + listen_for_messages_from_child(child_process) end) child_process end +# ╔═║ 7da416d1-5dfb-4510-9d32-62c1464a83d4 +function from_binary(message) + Serialization.deserialize(IOBuffer(message)) +end + +# ╔═║ 009ad714-b759-420a-b49a-6caed7ee3faf +function Base.iterate(message_reader::ReadMessages, state=nothing) + if eof(message_reader.io) + return nothing + end + + message = from_binary(read_message(message_reader.io)) + (message, nothing) +end + # ╔═║ 5fb65aec-3512-4f48-98ce-300ab9fdadfe begin function Base.iterate(channel::ChildChannel) @@ -498,11 +531,10 @@ end # ╔═║ 46d905fc-d41e-4c9a-a808-14710f64293a begin - function handle_message_from_parent_channel(; + function handle_message_from_parent_channel( parent_channel::ParentChannel, message::ChannelTakeMessage, ) - @info "Channel message taken!" next = iterate(parent_channel.result_channel) if next === nothing respond_to_parent( @@ -521,7 +553,7 @@ begin end end - function handle_message_from_parent_channel(; + function handle_message_from_parent_channel( parent_channel::ParentChannel, message::ChannelCloseMessage, ) @@ -531,7 +563,7 @@ begin false end - function handle_message_from_parent_channel(; + function handle_message_from_parent_channel( parent_channel::ParentChannel, message::Message, ) @@ -550,10 +582,9 @@ function create_parent_channel(; process, channel_id, result_channel) Channel(Inf) do input_channel try for message in input_channel - @info "Applying message" message - @timed if handle_message_from_parent_channel( - parent_channel=parent_channel, - message=message, + if handle_message_from_parent_channel( + parent_channel, + message, ) break end @@ -577,7 +608,7 @@ end # ╔═║ 38ca1c5b-e27e-4458-98a7-786b2d302ba3 begin - function handle_message_from_parent(; + function handle_message_from_parent( process::ParentProcess, channel_id::UUID, message @@ -585,20 +616,20 @@ begin @warn "Unknown message type from parent" message end - function handle_message_from_parent(; + function handle_message_from_parent( process::ParentProcess, channel_id::UUID, message::CreateChildChannelMessage ) result_channel = Main.eval(message.expr) - channels[channel_id] = create_parent_channel( + process.channels[channel_id] = create_parent_channel( process=process, channel_id=channel_id, result_channel=result_channel, ) end - function handle_message_from_parent(; + function handle_message_from_parent( process::ParentProcess, channel_id::UUID, message::ChannelCloseMessage @@ -608,7 +639,7 @@ begin end end - function handle_message_from_parent(; + function handle_message_from_parent( process::ParentProcess, channel_id::UUID, message::ChannelTakeMessage @@ -631,9 +662,7 @@ function listen_for_messages_from_parent(parent_process::ParentProcess) locks = Dict{UUID, ReentrantLock}() - @info "Well..." parent_process.parent_to_child for envelope in ReadMessages(parent_process.parent_to_child) - @info "Message from parent!" envelope channel_lock = get!(locks, envelope.channel_id) do ReentrantLock() end @@ -641,9 +670,9 @@ function listen_for_messages_from_parent(parent_process::ParentProcess) lock(channel_lock) try handle_message_from_parent( - process=parent_process, - channel_id=envelope.channel_id, - message=envelope.message, + parent_process, + envelope.channel_id, + envelope.message, ) catch error respond_to_parent( @@ -656,15 +685,8 @@ function listen_for_messages_from_parent(parent_process::ParentProcess) end end) end - @warn "oh no" end -# ╔═║ 1f868b3c-df9a-4d4d-a6a9-9a196298a3af -md"---" - -# ╔═║ bd530d27-ad17-45ef-8d8b-4b9e956f438d - - # ╔═║ 00000000-0000-0000-0000-000000000001 PLUTO_PROJECT_TOML_CONTENTS = """ [deps] @@ -692,17 +714,20 @@ uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" """ # ╔═║ Cell order: +# β•Ÿβ”€4c4b77dc-8545-4922-9897-110fa67c99f4 +# β•Ÿβ”€d3fe8144-6ba8-4dd9-b0e3-941a96422267 +# ╠═4b42e233-1f06-49c9-8c6a-9dc21c21ffb7 +# ╠═e3e16a8b-7124-4678-8fe7-12ed449e1954 +# β•Ÿβ”€934d18d4-936e-4ee0-aa6e-86aa6f66774c +# β•Ÿβ”€54a03cba-6d00-4632-8bd6-c60753c15ae6 # ╠═6925ffbb-fd9e-402c-a483-c78f28f892a5 # ╠═8c97d5cb-e346-4b94-b2a1-4fb5ff093bd5 # ╠═63531683-e295-4ba6-811f-63b0d384ba0f +# ╠═9ba8ee0c-c80c-41d9-8739-11e6ef5d3c15 +# ╠═4289b4ba-45f2-4be7-b983-68f7d97510fa # ╠═e6b3d1d9-0245-4dc8-a0a5-5831c254479b -# β•Ÿβ”€a6e50946-dd3d-4738-adc1-26534e184776 -# ╠═131a123b-e319-4939-90bf-7fb035ab2e75 -# ╠═e393ff80-3995-11ec-1217-b3642a509067 -# ╠═faf3c68e-d0fb-4dd9-ac1b-1ff8d922134b -# β•Ÿβ”€52495855-a52d-4edf-9c7c-811a5060e641 -# β•Ÿβ”€7da416d1-5dfb-4510-9d32-62c1464a83d4 -# β•Ÿβ”€87612815-7981-4a69-a65b-4465f48c9fd9 +# ╠═2a538f97-a9d1-4dcf-a098-5b1e5a8df3ae +# ╠═87612815-7981-4a69-a65b-4465f48c9fd9 # ╠═009ad714-b759-420a-b49a-6caed7ee3faf # β•Ÿβ”€de680a32-e053-4655-a1cd-111d81a037e6 # ╠═ce2acdd1-b4bb-4a8f-8346-a56dfa55f56b @@ -716,27 +741,18 @@ uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" # β•Ÿβ”€17cb5a65-2a72-4e7d-8fba-901452b2c19f # ╠═85920c27-d8a6-4b5c-93b6-41daa5866f9d # ╠═af4f0720-f7f7-4d8a-bd8c-8cf5abaf10a0 +# ╠═368ccd31-1681-4a4a-a103-2b9afc0813ee # ╠═d4da756e-fa81-49d8-8c39-d76f9d15e96f # ╠═e787d8bd-499d-4a41-8913-62b3d9346748 # ╠═20f66652-bca0-4e47-a4b7-502cfbcb3db5 # ╠═6a7aa0ce-0ae3-4e7d-a93a-bfdebe406220 # ╠═f99f4659-f71e-4f2e-a674-67ba69289817 -# ╠═3a62b419-eca2-4de1-89a4-fc9ad6f68372 -# ╠═61410bd1-8456-4608-92d9-3c863f65b89c -# ╠═409707ff-b1ea-453d-b933-0a4b1e5f44c8 -# ╠═0feb758e-38d4-48c0-a5e9-9d1129b1b1b2 # ╠═2342d663-030f-4ed2-b1d5-5be1910b6d4c # ╠═abe897ff-6e86-40eb-a1a6-2918b6c3c5a7 -# ╠═4b42e233-1f06-49c9-8c6a-9dc21c21ffb7 -# ╠═e3e16a8b-7124-4678-8fe7-12ed449e1954 -# ╠═934d18d4-936e-4ee0-aa6e-86aa6f66774c -# ╠═b48f36d1-e3d2-408e-a705-93e609f2fd04 -# ╠═54a03cba-6d00-4632-8bd6-c60753c15ae6 -# ╠═d3fe8144-6ba8-4dd9-b0e3-941a96422267 # β•Ÿβ”€884e103a-8925-477d-a264-f15d02a49aa9 # ╠═b8865d63-19fa-4438-87a2-ccb531bd73a4 # ╠═e1e79669-8d0a-4b04-a7fd-469e7d8e65b1 -# ╠═d3ce3ecf-c1e4-4484-ac38-b332a4b13034 +# ╠═ed6c16ed-bf4a-40ce-9c14-a72fd77e24a1 # ╠═5fb65aec-3512-4f48-98ce-300ab9fdadfe # ╠═fc0ca4b5-e03b-4c08-b43d-913ee12269c7 # ╠═adfb2d12-07a0-4d5e-8e24-1676c07107c7 @@ -747,6 +763,17 @@ uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" # ╠═441c27e4-6884-4887-9cd5-bc55d0c49760 # ╠═f81ff6f7-f230-4373-b60b-76e8a0eba929 # β•Ÿβ”€1f868b3c-df9a-4d4d-a6a9-9a196298a3af -# ╠═bd530d27-ad17-45ef-8d8b-4b9e956f438d +# β•Ÿβ”€77df6f7b-ed8e-442a-9489-873fc392937c +# ╠═3a62b419-eca2-4de1-89a4-fc9ad6f68372 +# ╠═61410bd1-8456-4608-92d9-3c863f65b89c +# ╠═409707ff-b1ea-453d-b933-0a4b1e5f44c8 +# ╠═0feb758e-38d4-48c0-a5e9-9d1129b1b1b2 +# ╠═b0bcdbdf-a043-4d25-8582-ed173006e040 +# β•Ÿβ”€a6e50946-dd3d-4738-adc1-26534e184776 +# β•Ÿβ”€131a123b-e319-4939-90bf-7fb035ab2e75 +# β•Ÿβ”€e393ff80-3995-11ec-1217-b3642a509067 +# β•Ÿβ”€faf3c68e-d0fb-4dd9-ac1b-1ff8d922134b +# β•Ÿβ”€52495855-a52d-4edf-9c7c-811a5060e641 +# β•Ÿβ”€7da416d1-5dfb-4510-9d32-62c1464a83d4 # β•Ÿβ”€00000000-0000-0000-0000-000000000001 # β•Ÿβ”€00000000-0000-0000-0000-000000000002 diff --git a/src/evaluation/ChildProcessesTest.jl b/src/evaluation/ChildProcessesTest.jl index 5239204b41..84b3bf83f9 100644 --- a/src/evaluation/ChildProcessesTest.jl +++ b/src/evaluation/ChildProcessesTest.jl @@ -1,5 +1,5 @@ ### A Pluto.jl notebook ### -# v0.17.0 +# v0.17.7 using Markdown using InteractiveUtils @@ -7,8 +7,9 @@ using InteractiveUtils # This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error). macro bind(def, element) quote + local iv = try Base.loaded_modules[Base.PkgId(Base.UUID("6e696c72-6542-2067-7265-42206c756150"), "AbstractPlutoDingetjes")].Bonds.initial_value catch; b -> missing; end local el = $(esc(element)) - global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : missing + global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : iv(el) el end end @@ -19,14 +20,14 @@ import Serialization # ╔═║ d30e1e1b-fa6f-4fbc-bccb-1fcfc7b829df import UUIDs: UUID, uuid4 -# ╔═║ a901d255-c18c-45ed-9827-afd79246613c -module PlutoHooks include("./PlutoHooks.jl") end - # ╔═║ f7d14367-27d7-41a5-9f6a-79cf5e721a7d import PlutoUI +# ╔═║ fe669218-18c3-46c7-80e8-7b1ab6fa77d2 +import PlutoLinks: @ingredients, @use_task + # ╔═║ f9675ee0-e728-420b-81bd-22e57583c587 -import .PlutoHooks: @use_effect, @use_ref, @use_state, @background, @use_memo +import PlutoHooks: @use_effect, @use_ref, @use_state, @use_memo # ╔═║ ca96e0d5-0904-4ae5-89d0-c1a9187710a1 Base.@kwdef struct PlutoProcess @@ -41,15 +42,6 @@ function Base.show(io::IO, ::MIME"text/plain", process::PlutoProcess) write(io, process_running(process.process) ? "Running" : "Stopped") end -# ╔═║ c9a9a089-037b-4fa8-91b2-980a318faee7 -begin - Frames = PlutoHooks.@ingredients("./SerializationLibrary.jl") - ChildProcesses = Frames - Main.eval(quote - var"SerializationLibrary.jl" = $(Frames) - end) -end - # ╔═║ 1a2f116b-78f7-47b6-b96b-c0c74b5c5a35 @bind reset_process_counter PlutoUI.CounterButton("Reset process!") @@ -59,6 +51,98 @@ end # ╔═║ 09f4b4b6-75a4-4727-9405-e4fc45c2cda7 import Distributed +# ╔═║ ffc98725-a88f-4dac-b387-f73cfd07510a +# @use_task([x]) do +# ChildProcesses.call(x, quote +# 1 + 1 +# end) +# end + +# ╔═║ a26cc458-4578-427f-840d-71d78c5c8b01 +begin + ChildProcesses = @ingredients("./ChildProcesses.jl").ChildProcesses + var"ChildProcesses.jl" = ChildProcesses +end + +# ╔═║ 730cfe5e-1541-4c08-8d4a-86d1f9e4115e +let + process = ChildProcesses.create_child_process() + + ChildProcesses.call(process, :(throw("hi"))) +end + +# ╔═║ b7ac2fea-41f4-47f3-a294-944378cb7093 +x = @use_memo([]) do + ChildProcesses.create_child_process() +end + +# ╔═║ 872a8651-736c-47dd-80e6-b645fb0490c9 +kill(x, Base.SIGINT) + +# ╔═║ c1038e6a-c4e9-4b79-b3f9-35a9f64b0f64 +task = @use_task([x]) do + ChildProcesses.call(x, quote + while true; end + end) +end + +# ╔═║ 75fc343a-7904-45f8-be09-a542255ba673 +istaskdone(task) ? fetch(task) : nothing + +# ╔═║ 281b4aab-307a-4d90-9dfb-f422b9567736 +# process_output = let +# # error("Nope") + +# my_stderr = @use_memo([reset_process_counter, ChildProcesses]) do +# Pipe() +# end +# output, set_output = @use_state("") + +# process = @use_memo([my_stderr]) do +# ChildProcesses.create_child_process( +# custom_stderr=my_stderr, +# exeflags=["--color=yes", "--threads=4"], +# ) +# end +# @use_effect([process]) do +# return () -> begin +# kill(process) +# end +# end + +# # So we re-run the whole thing when the process exists +# _, refresh_state = @use_state(nothing) +# @use_task([]) do +# if process_running(process) +# wait(process) +# refresh_state(nothing) +# end +# end + +# @use_task([]) do +# while process_running(process) && !eof(my_stderr) +# new_output = String(readavailable(my_stderr)) +# set_output((output) -> begin +# output * new_output +# end) +# end +# end + +# pluto_process = @use_memo([process, my_stderr]) do +# PlutoProcess( +# process=process, +# stderr=my_stderr, +# stdin=process.process.in, +# stdout=process.process.out, +# ) +# end + +# PlutoUI.with_terminal() do +# print(output) +# pluto_process +# end +# end + # ╔═║ e5edaa4d-74ff-4f6e-a045-71fd5494dd79 # @use_memo([spawned_process]) do # try @@ -102,15 +186,6 @@ import Distributed # end) # end -# ╔═║ 33bcb6d2-5aea-4b7d-b5c4-e0dd2abc73f1 -1 + 1 - -# ╔═║ 2db3c415-e827-4ecb-a2c9-f407650c37ac -@bind g PlutoUI.Slider(1:100) - -# ╔═║ 4ac2d47a-30b5-4095-a873-258b003539cf -h = g + 2 - # ╔═║ de602aaa-704c-45c4-8b7b-fc58e41236ce begin struct FakeProcess <: Base.AbstractPipe @@ -127,81 +202,60 @@ end # Base.readbytes!(PipeBuffer(), UInt8[], 10) # end -# ╔═║ f8bf6cc4-5ccd-4f0d-b7de-0f192bb3dfb1 -buffer = IOBuffer() - -# ╔═║ c96496a2-50bd-475c-ae02-f9b828080610 -@which write(buffer, UInt8[]) - -# ╔═║ 25a3fc9c-a96d-4077-a304-c59554e8f568 -@which unsafe_write(buffer, pointer(UInt8[]), sizeof(UInt8[])) - # ╔═║ ec4a5558-28dd-47c1-b015-8799d9cb8800 -function Base.eof(buffer::IOBuffer) - while buffer.readable - if bytesavailable(buffer) !== 0 - return false - end - # eval(:(@info "OOPS")) - sleep(1) - yield() - end - return true -end +# function Base.eof(buffer::IOBuffer) +# while buffer.readable +# if bytesavailable(buffer) !== 0 +# return false +# end +# # eval(:(@info "OOPS")) +# sleep(1) +# yield() +# end +# return true +# end -# ╔═║ 281b4aab-307a-4d90-9dfb-f422b9567736 -process_output = let - error("Nope") - - my_stderr = @use_memo([reset_process_counter, ChildProcesses]) do - Pipe() - end - output, set_output = @use_state("", [my_stderr]) +# ╔═║ ce384249-52c5-47d8-9c93-18837432b625 +# let +# parent_to_child = PipeBuffer() +# child_to_parent = PipeBuffer() +# child_process = @use_memo([ChildProcesses.ChildProcess, FakeProcess]) do +# process = FakeProcess(child_to_parent, parent_to_child) +# child_process = ChildProcesses.ChildProcess(process=process) +# end +# parent_process = @use_memo([ChildProcesses.ParentProcess]) do +# ChildProcesses.ParentProcess( +# parent_to_child=parent_to_child, +# child_to_parent=child_to_parent, +# ) +# end - process = @use_memo([my_stderr]) do - ChildProcesses.create_child_process( - custom_stderr=my_stderr, - exeflags=["--color=yes", "--threads=4"], - ) - end - @use_effect([process]) do - return () -> begin - kill(process) - end - end +# @info "#1" +# @use_task([ChildProcesses.start_from_child_loop, child_process]) do +# ChildProcesses.start_from_child_loop(child_process) +# end +# @info "#2" +# @use_task([ChildProcesses.listen_for_messages_from_parent, parent_process]) do +# ChildProcesses.listen_for_messages_from_parent(parent_process) +# end +# @info "#3" - # So we re-run the whole thing when the process exists - _, refresh_state = @use_state(nothing, [process]) - @background begin - if process_running(process) - wait(process) - refresh_state(nothing) - end - end +# result, set_result = @use_state(Pending()) - @background begin - while process_running(process) && !eof(my_stderr) - new_output = String(readavailable(my_stderr)) - set_output((output) -> begin - output * new_output - end) - end - end +# @use_task([ChildProcesses.call, child_process]) do +# try +# result = ChildProcesses.call(child_process, quote +# 1 + 1 +# end) - pluto_process = @use_memo([process, my_stderr]) do - PlutoProcess( - process=process, - stderr=my_stderr, - stdin=process.process.in, - stdout=process.process.out, - ) - end +# set_result(Result(result)) +# catch error +# set_result(Failure(error)) +# end +# end - PlutoUI.with_terminal(show_value=true) do - print(output) - pluto_process - end -end +# result +# end # ╔═║ 4be7e097-72a3-4590-bcfb-a7dacb78159c spawned_process = process_output.value.process; @@ -212,14 +266,22 @@ md"---" # ╔═║ 95c5a5bc-db23-4ad3-8ae8-81bc8f0edfd4 import BenchmarkTools +# ╔═║ 4541f52f-1217-4dcd-b44c-042c7ca246bd +1 + 1 + # ╔═║ ced9d1e9-7075-4ff2-8ca2-6a349f2a69c4 -# let -# stream = PipeBuffer() -# BenchmarkTools.@benchmark begin -# ChildProcesses.send_message($stream, ChildProcesses.to_binary(Dict(:x => 1))) -# ChildProcesses.from_binary(ChildProcesses.read_message($stream)) -# end -# end +let + stream = PipeBuffer() + BenchmarkTools.@benchmark begin + input = Dict(:x => 1) + ChildProcesses.send_message($stream, ChildProcesses.to_binary(input)) + output = ChildProcesses.from_binary(ChildProcesses.read_message($stream)) + + if input != output + throw("Waoh, input and output should match but didn't!") + end + end +end # ╔═║ be18d157-6b55-4eaa-99fe-c398e992a9fa md"### @task_result" @@ -233,66 +295,21 @@ struct Result value end # ╔═║ 83540498-f317-4d8b-8dc6-80a93247b3b2 struct Failure error end -# ╔═║ 78803cff-2af8-4266-8fad-6911faf17910 -macro task_result(expr, deps=nothing) - quote - result, set_result = @use_state($(Pending)()) - - @background($(esc(deps))) do - try - result = $(esc(expr))() - set_result($(Result)(result)) - catch error - set_result($(Failure)(error)) - end - end - - result - end -end - -# ╔═║ ce384249-52c5-47d8-9c93-18837432b625 -let - parent_to_child = PipeBuffer() - child_to_parent = PipeBuffer() - child_process = @use_memo([ChildProcesses.ChildProcess, FakeProcess]) do - process = FakeProcess(child_to_parent, parent_to_child) - child_process = ChildProcesses.ChildProcess(process=process) - end - parent_process = @use_memo([ChildProcesses.ParentProcess]) do - ChildProcesses.ParentProcess( - parent_to_child=parent_to_child, - child_to_parent=child_to_parent, - ) - end - - @info "#1" - @background([ChildProcesses.start_from_child_loop, child_process]) do - ChildProcesses.start_from_child_loop(child_process) - end - @info "#2" - @background([ChildProcesses.listen_for_messages_from_parent, parent_process]) do - ChildProcesses.listen_for_messages_from_parent(parent_process) - end - @info "#3" - @task_result([ChildProcesses.call, child_process]) do - ChildProcesses.call(child_process, quote - 1 + 1 - end) - end -end - # ╔═║ 00000000-0000-0000-0000-000000000001 PLUTO_PROJECT_TOML_CONTENTS = """ [deps] BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" +PlutoHooks = "0ff47ea0-7a50-410d-8455-4348d5de0774" +PlutoLinks = "0ff47ea0-7a50-410d-8455-4348d5de0420" PlutoUI = "7f904dfe-b85e-4ff6-b463-dae2292396a8" Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] BenchmarkTools = "~1.2.0" +PlutoHooks = "~0.0.3" +PlutoLinks = "~0.1.1" PlutoUI = "~0.7.16" """ @@ -317,6 +334,9 @@ uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" deps = ["Random", "Serialization", "Sockets"] uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" +[[FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" + [[Hyperscript]] deps = ["Test"] git-tree-sha1 = "8d511d5b81240fc8e6802386302675bdf47737b9" @@ -367,6 +387,18 @@ git-tree-sha1 = "d911b6a12ba974dabe2291c6d450094a7226b372" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" version = "2.1.1" +[[PlutoHooks]] +deps = ["FileWatching", "InteractiveUtils", "Markdown", "UUIDs"] +git-tree-sha1 = "f297787f7d7507dada25f6769fe3f08f6b9b8b12" +uuid = "0ff47ea0-7a50-410d-8455-4348d5de0774" +version = "0.0.3" + +[[PlutoLinks]] +deps = ["FileWatching", "InteractiveUtils", "Markdown", "PlutoHooks", "UUIDs"] +git-tree-sha1 = "5f45fc68dd9eb422358a8008e3fb8df3c01d8ab8" +uuid = "0ff47ea0-7a50-410d-8455-4348d5de0420" +version = "0.1.1" + [[PlutoUI]] deps = ["Base64", "Dates", "Hyperscript", "HypertextLiteral", "IOCapture", "InteractiveUtils", "JSON", "Logging", "Markdown", "Random", "Reexport", "UUIDs"] git-tree-sha1 = "4c8a7d080daca18545c56f1cac28710c362478f3" @@ -422,38 +454,38 @@ uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" # ╔═║ Cell order: # ╠═7e613bd2-616a-4687-8af5-a22c7a747d97 # ╠═d30e1e1b-fa6f-4fbc-bccb-1fcfc7b829df -# ╠═a901d255-c18c-45ed-9827-afd79246613c # ╠═f7d14367-27d7-41a5-9f6a-79cf5e721a7d +# ╠═fe669218-18c3-46c7-80e8-7b1ab6fa77d2 # ╠═f9675ee0-e728-420b-81bd-22e57583c587 -# ╠═c9a9a089-037b-4fa8-91b2-980a318faee7 # β•Ÿβ”€ca96e0d5-0904-4ae5-89d0-c1a9187710a1 -# β•Ÿβ”€b7734627-ba2a-48d8-8fd8-a5c94716da20 +# ╠═b7734627-ba2a-48d8-8fd8-a5c94716da20 # β•Ÿβ”€1a2f116b-78f7-47b6-b96b-c0c74b5c5a35 # β•Ÿβ”€8d6f46b6-e6cf-4094-a32a-f1b13dc005f6 # ╠═09f4b4b6-75a4-4727-9405-e4fc45c2cda7 +# ╠═730cfe5e-1541-4c08-8d4a-86d1f9e4115e +# ╠═b7ac2fea-41f4-47f3-a294-944378cb7093 +# ╠═c1038e6a-c4e9-4b79-b3f9-35a9f64b0f64 +# ╠═75fc343a-7904-45f8-be09-a542255ba673 +# ╠═ffc98725-a88f-4dac-b387-f73cfd07510a +# ╠═872a8651-736c-47dd-80e6-b645fb0490c9 +# ╠═a26cc458-4578-427f-840d-71d78c5c8b01 # ╠═281b4aab-307a-4d90-9dfb-f422b9567736 # ╠═e5edaa4d-74ff-4f6e-a045-71fd5494dd79 # ╠═5fb236c3-b67d-47ee-8644-84bd51e577b1 # ╠═8bd24b7b-4837-46b7-a6e9-b674630c2f56 # ╠═e33a9b31-722e-425e-be5e-b33517bec8e3 -# ╠═33bcb6d2-5aea-4b7d-b5c4-e0dd2abc73f1 -# ╠═2db3c415-e827-4ecb-a2c9-f407650c37ac -# ╠═4ac2d47a-30b5-4095-a873-258b003539cf # ╠═de602aaa-704c-45c4-8b7b-fc58e41236ce # ╠═abf694f4-c5c4-4a4f-b30f-62e358149195 -# ╠═f8bf6cc4-5ccd-4f0d-b7de-0f192bb3dfb1 -# ╠═c96496a2-50bd-475c-ae02-f9b828080610 -# ╠═25a3fc9c-a96d-4077-a304-c59554e8f568 # ╠═ec4a5558-28dd-47c1-b015-8799d9cb8800 # ╠═ce384249-52c5-47d8-9c93-18837432b625 # ╠═4be7e097-72a3-4590-bcfb-a7dacb78159c -# ╠═49fdc8a3-0e1a-42f0-acc4-b823eec91d31 +# β•Ÿβ”€49fdc8a3-0e1a-42f0-acc4-b823eec91d31 # ╠═95c5a5bc-db23-4ad3-8ae8-81bc8f0edfd4 +# ╠═4541f52f-1217-4dcd-b44c-042c7ca246bd # ╠═ced9d1e9-7075-4ff2-8ca2-6a349f2a69c4 # β•Ÿβ”€be18d157-6b55-4eaa-99fe-c398e992a9fa # ╠═12578b59-0161-4e72-afef-825166a62121 # ╠═34d90560-5a3e-4c7f-8126-35e1a6153aa1 # ╠═83540498-f317-4d8b-8dc6-80a93247b3b2 -# ╠═78803cff-2af8-4266-8fad-6911faf17910 # β•Ÿβ”€00000000-0000-0000-0000-000000000001 # β•Ÿβ”€00000000-0000-0000-0000-000000000002 diff --git a/src/evaluation/WorkspaceManager.jl b/src/evaluation/WorkspaceManager.jl index e04ce330ba..e9cabb97c9 100644 --- a/src/evaluation/WorkspaceManager.jl +++ b/src/evaluation/WorkspaceManager.jl @@ -238,7 +238,27 @@ function distributed_exception_result(ex::Base.IOError, workspace::Workspace) ) end +function distributed_exception_result(ex::ChildProcesses.ChildProcessException, workspace::Workspace) + ( + output_formatted=PlutoRunner.format_output(CapturedException(InterruptException(), [])), + errored=true, + interrupted=true, + process_exited=false, + runtime=nothing, + published_objects=Dict{String,Any}(), + ) +end +function distributed_exception_result(ex::ChildProcesses.ProcessExitedException, workspace::Workspace) + ( + output_formatted=PlutoRunner.format_output(CapturedException(ex, [])), + errored=true, + interrupted=true, + process_exited=true && !workspace.discarded, # don't report a process exit if the workspace was discarded on purpose + runtime=nothing, + published_objects=Dict{String,Any}(), + ) +end function distributed_exception_result(exs::CompositeException, workspace::Workspace) ex = exs.exceptions |> first @@ -456,7 +476,7 @@ function interrupt_workspace(session_notebook::Union{SN,Workspace}; verbose=true # TODO: this will also kill "pending" evaluations, and any evaluations started within 100ms of the kill. A global "evaluation count" would fix this. # TODO: listen for the final words of the remote process on stdout/stderr: "Force throwing a SIGINT" try - verbose && @info "Sending interrupt to process $(workspace.pid)" + verbose && @info "Sending interrupt to process $(workspace)..." kill(workspace.process, Base.SIGINT) if poll(() -> isready(workspace.dowork_token), 5.0, 5/100) @@ -481,8 +501,7 @@ function interrupt_workspace(session_notebook::Union{SN,Workspace}; verbose=true true catch ex if !(ex isa KeyError) - @warn "Interrupt failed for unknown reason" - showerror(ex, stacktrace(catch_backtrace())) + @warn "Interrupt failed for unknown reason" sprint(showerror, ex, stacktrace(catch_backtrace())) end false end From e6a692c131cd450f02f29b577f3f66b8ecc83b05 Mon Sep 17 00:00:00 2001 From: Michiel Dral Date: Tue, 25 Jan 2022 22:05:36 +0100 Subject: [PATCH 25/28] :shrug: --- src/evaluation/ChildProcessesNotebook.jl | 400 ++++++++++++----------- src/evaluation/ChildProcessesTest.jl | 277 ++-------------- 2 files changed, 235 insertions(+), 442 deletions(-) diff --git a/src/evaluation/ChildProcessesNotebook.jl b/src/evaluation/ChildProcessesNotebook.jl index cf177653d5..4e6b00afe5 100644 --- a/src/evaluation/ChildProcessesNotebook.jl +++ b/src/evaluation/ChildProcessesNotebook.jl @@ -7,6 +7,20 @@ using InteractiveUtils # ╔═║ 4c4b77dc-8545-4922-9897-110fa67c99f4 md"# ChildProcesses" +# ╔═║ 934d18d4-936e-4ee0-aa6e-86aa6f66774c +juliapath() = joinpath(Sys.BINDIR::String, Base.julia_exename()) + +# ╔═║ 4df77979-0c35-4139-aa5e-da43c1de3a77 +if VERSION >= v"1.7.0" + function get_task_error(t::Task) + throw("TODO") + end +else + function get_task_error(t::Task) + CapturedException(Base.catch_stack(t)[end]...) + end +end + # ╔═║ 6925ffbb-fd9e-402c-a483-c78f28f892a5 import Serialization @@ -28,20 +42,22 @@ Base.@kwdef mutable struct ParentProcess parent_to_child=stdin child_to_parent=stdout channels::Dict{UUID,AbstractChannel}=Dict{UUID,AbstractChannel}() - lock=ReentrantLock() + messaging_lock=ReentrantLock() end # ╔═║ 87182605-f5a1-46a7-968f-bdfb9d0e08fa function setup_parent_process() - real_stdout = stdout # Redirect things written to stdout to stderr, # stderr will showup in the Pluto with_terminal view. + real_stdout = stdout redirect_stdout(stderr) - # TODO Maybe redirect stdin too, just to be sure? + # Just take away stdin, just in case + real_stdin = stdin + redirect_stdin() ParentProcess( - parent_to_child=stdin, + parent_to_child=real_stdin, child_to_parent=real_stdout, ) end @@ -58,11 +74,33 @@ md""" ## Parent process side """ +# ╔═║ abe897ff-6e86-40eb-a1a6-2918b6c3c5a7 +struct LocalSandbox end + +# ╔═║ 884e103a-8925-477d-a264-f15d02a49aa9 +md""" +## ChildChannel + +I guess this is similar to Distributed.ChildChannel. +It communicates all actions that happen on it (take!, put!, close) to the other side. There is a local channel stored that is used as buffer. +""" + +# ╔═║ 1d63f09f-dfb2-40dd-beb5-45125cb19006 +md"## Message types" + +# ╔═║ fc0ca4b5-e03b-4c08-b43d-913ee12269c7 +abstract type Message end + +# ╔═║ 3431051e-55ce-46c1-a0f5-364662f5c77b +MessageChannel = AbstractChannel{Message} + # ╔═║ 85920c27-d8a6-4b5c-93b6-41daa5866f9d Base.@kwdef mutable struct ChildProcess process::Base.AbstractPipe - channels::Dict{UUID,AbstractChannel}=Dict{UUID,AbstractChannel}() - lock=ReentrantLock() + channels::Dict{UUID,MessageChannel}=Dict{UUID,MessageChannel}() + + "Lock to make sure there are no messages being sent intervowen" + messaging_lock=ReentrantLock() end # ╔═║ 63531683-e295-4ba6-811f-63b0d384ba0f @@ -99,20 +137,6 @@ Base.process_running(p::ChildProcess) = Base.process_running(p.process) # ╔═║ e787d8bd-499d-4a41-8913-62b3d9346748 Base.wait(process::ChildProcess) = Base.wait(process.process) -# ╔═║ abe897ff-6e86-40eb-a1a6-2918b6c3c5a7 -struct LocalSandbox end - -# ╔═║ 934d18d4-936e-4ee0-aa6e-86aa6f66774c -juliapath() = joinpath(Sys.BINDIR::String, Base.julia_exename()) - -# ╔═║ 884e103a-8925-477d-a264-f15d02a49aa9 -md""" -## ChildChannel - -I guess this is similar to Distributed.ChildChannel. -It communicates all actions that happen on it (take!, put!, close) to the other side. There is a local channel stored that is used as buffer. -""" - # ╔═║ b8865d63-19fa-4438-87a2-ccb531bd73a4 Base.@kwdef struct ChildChannel{T} <: AbstractChannel{T} process::ChildProcess @@ -120,8 +144,11 @@ Base.@kwdef struct ChildChannel{T} <: AbstractChannel{T} local_channel::AbstractChannel{T} end -# ╔═║ fc0ca4b5-e03b-4c08-b43d-913ee12269c7 -abstract type Message end +# ╔═║ f81ff6f7-f230-4373-b60b-76e8a0eba929 +Base.@kwdef struct Envelope + channel_id::UUID + message::Message +end # ╔═║ adfb2d12-07a0-4d5e-8e24-1676c07107c7 struct CreateChildChannelMessage <: Message @@ -135,7 +162,7 @@ end # ╔═║ f9270f05-fa22-4027-a2ca-7db61d057c56 struct ErrorMessage <: Message - error::Exception + error::CapturedException end # ╔═║ 0ba318bc-7b4c-4d0f-a46c-2f9dca756926 @@ -151,12 +178,6 @@ end struct ChannelTakeMessage <: Message end -# ╔═║ f81ff6f7-f230-4373-b60b-76e8a0eba929 -Base.@kwdef struct Envelope - channel_id::UUID - message::Message -end - # ╔═║ 1f868b3c-df9a-4d4d-a6a9-9a196298a3af md"---" @@ -254,13 +275,12 @@ function respond_to_parent(; to::ParentProcess, about::UUID, with::Message) channel_id=about, message=with, )) - lock(to.lock) + lock(to.messaging_lock) try send_message(to.child_to_parent, binary_message) finally - unlock(to.lock) + unlock(to.messaging_lock) end - end # ╔═║ 20f66652-bca0-4e47-a4b7-502cfbcb3db5 @@ -269,12 +289,12 @@ function send_message_without_response(process::ChildProcess, envelope::Envelope throw(ProcessExitedException(process=process)) end - lock(process.lock) + lock(process.messaging_lock) try send_message(process.process, to_binary(envelope)) - return envelope + nothing finally - unlock(process.lock) + unlock(process.messaging_lock) end end @@ -315,6 +335,98 @@ end # ╔═║ 4289b4ba-45f2-4be7-b983-68f7d97510fa Base.close(process::ChildProcess) = Base.close(process.process) +# ╔═║ 2342d663-030f-4ed2-b1d5-5be1910b6d4c +function create_channel(fn, process::ChildProcess, message) + remote_channel = create_channel(process, message) + + try + return fn(remote_channel) + finally + close(remote_channel) + end +end + +# ╔═║ 54a03cba-6d00-4632-8bd6-c60753c15ae6 +function listen_for_messages_from_child(child_process) + try + for result in ReadMessages(child_process.process) + if !(result isa Envelope && result.channel_id !== nothing) + throw("Huh") + end + + + if haskey(child_process.channels, result.channel_id) + message = result.message + channel = child_process.channels[result.channel_id] + put!(channel, message) + else + throw("No channel to push to") + end + end + catch error + @error "Error in main processing" error bt=stacktrace(catch_backtrace()) + close(child_process.process) + finally + for (channel_id, channel) in collect(child_process.channels) + close(channel, ProcessExitedException(process=child_process)) + end + end +end + +# ╔═║ d3fe8144-6ba8-4dd9-b0e3-941a96422267 +""" + create_child_process(; custom_stderr=stderr, exeflags=["--history-file=no"]) + +Spawn a child process that you'll be able to `call()` on and communicate with over +`RemoteChannels`. + +It spawns a process using `open` with some ad-hoc julia code that also loads the `ChildProcesses` module. It then spawns a loop for listening to the spawned process. +The returned `ChildProcess` is basically a bunch of event listeners (in the form of channels) that +""" +function create_child_process(; custom_stderr=stderr, exeflags=["--history-file=no", "--threads=auto"]) + this_file = split(@__FILE__(), "#")[1] + this_module = string(nameof(@__MODULE__)) + + code = """ + # Make sure this library is included in main + var"$this_module" = @eval Main module var"$this_module" + include("$this_file") + end + + const parent_process = var"$this_module".setup_parent_process() + + Threads.@spawn begin + try + var"$this_module".listen_for_messages_from_parent(parent_process) + @info "Done?" + catch error + @error "Shutdown error" error + rethrow(error) + end + end + + try + while true + sleep(typemax(UInt64)) + end + catch e + @error "HMMM SIGNINT MAYBE?" e + end + """ + + process = open( + pipeline(`$(juliapath()) $exeflags -e $code`, stderr=custom_stderr), + read=true, + write=true, + ) + + child_process = ChildProcess(process=process) + schedule(Task() do + listen_for_messages_from_child(child_process) + end) + child_process +end + # ╔═║ 6a7aa0ce-0ae3-4e7d-a93a-bfdebe406220 begin function handle_child_message( @@ -350,7 +462,7 @@ begin input_channel::AbstractChannel, output_channel::AbstractChannel, ) - close(input_channel, ChildProcessException( + close(output_channel, ChildProcessException( process=process, captured=message.error, )) @@ -360,15 +472,19 @@ end # ╔═║ f99f4659-f71e-4f2e-a674-67ba69289817 function create_channel(process::ChildProcess, expr) channel_id = uuid4() - envelope = send_message_without_response(process, Envelope( - channel_id=channel_id, - message=CreateChildChannelMessage(expr) - )) + # Create and register channel for responses response_channel = Channel{Message}() process.channels[channel_id] = response_channel - actual_values_channes = Channel(Inf) do ch + # Tell the child process to start executing + send_message_without_response(process, Envelope( + channel_id=channel_id, + message=CreateChildChannelMessage(expr) + )) + + # Map the messages from the child process to actions + actual_values_channels = Channel(Inf) do ch try for message in response_channel handle_child_message( @@ -379,6 +495,7 @@ function create_channel(process::ChildProcess, expr) ) end catch e + @error "THIS SHOULDNT ERROR MATE" e close(ch, e) finally close(ch) @@ -389,21 +506,10 @@ function create_channel(process::ChildProcess, expr) ChildChannel( process=process, id=channel_id, - local_channel=actual_values_channes, + local_channel=actual_values_channels, ) end -# ╔═║ 2342d663-030f-4ed2-b1d5-5be1910b6d4c -function create_channel(fn, process::ChildProcess, message) - remote_channel = create_channel(process, message) - - try - return fn(remote_channel) - finally - close(remote_channel) - end -end - # ╔═║ 4b42e233-1f06-49c9-8c6a-9dc21c21ffb7 function call(process::ChildProcess, expr) create_channel(process, quote @@ -425,77 +531,6 @@ function call_without_fetch(process::ChildProcess, expr) end end -# ╔═║ 54a03cba-6d00-4632-8bd6-c60753c15ae6 -function listen_for_messages_from_child(child_process) - try - for result in ReadMessages(child_process.process) - if !(result isa Envelope && result.channel_id !== nothing) - throw("Huh") - end - - - if haskey(child_process.channels, result.channel_id) - message = result.message - channel = child_process.channels[result.channel_id] - put!(channel, message) - else - throw("No channel to push to") - end - end - catch error - @error "Error in main processing" error bt=stacktrace(catch_backtrace()) - close(child_process.process) - finally - for (channel_id, channel) in collect(child_process.channels) - close(channel, ProcessExitedException(process=child_process)) - end - end -end - -# ╔═║ d3fe8144-6ba8-4dd9-b0e3-941a96422267 -""" - create_child_process(; custom_stderr=stderr, exeflags=["--history-file=no"]) - -Spawn a child process that you'll be able to `call()` on and communicate with over -`RemoteChannels`. - -It spawns a process using `open` with some ad-hoc julia code that also loads the `ChildProcesses` module. It then spawns a loop for listening to the spawned process. -The returned `ChildProcess` is basically a bunch of event listeners (in the form of channels) that -""" -function create_child_process(; custom_stderr=stderr, exeflags=["--history-file=no"]) - this_file = split(@__FILE__(), "#")[1] - this_module = string(nameof(@__MODULE__)) - - code = """ - # Make sure this library is included in main - var"$this_module" = @eval Main module var"$this_module" - include("$this_file") - end - - const parent_process = var"$this_module".setup_parent_process() - - try - var"$this_module".listen_for_messages_from_parent(parent_process) - @info "Done?" - catch error - @error "Shutdown error" error - rethrow(error) - end - """ - - process = open( - pipeline(`$(juliapath()) $exeflags -e $code`, stderr=custom_stderr), - read=true, - write=true, - ) - - child_process = ChildProcess(process=process) - schedule(Task() do - listen_for_messages_from_child(child_process) - end) - child_process -end - # ╔═║ 7da416d1-5dfb-4510-9d32-62c1464a83d4 function from_binary(message) Serialization.deserialize(IOBuffer(message)) @@ -574,14 +609,15 @@ end # ╔═║ 13089c5d-f833-4fb7-b8cd-4158b1a57103 function create_parent_channel(; process, channel_id, result_channel) - parent_channel = ParentChannel( - process=process, - channel_id=channel_id, - result_channel=result_channel, - ) Channel(Inf) do input_channel try for message in input_channel + parent_channel = ParentChannel( + process=process, + channel_id=channel_id, + result_channel=result_channel, + ) + if handle_message_from_parent_channel( parent_channel, message, @@ -606,77 +642,51 @@ function create_parent_channel(; process, channel_id, result_channel) end end -# ╔═║ 38ca1c5b-e27e-4458-98a7-786b2d302ba3 -begin - function handle_message_from_parent( - process::ParentProcess, - channel_id::UUID, - message - ) - @warn "Unknown message type from parent" message - end - - function handle_message_from_parent( - process::ParentProcess, - channel_id::UUID, - message::CreateChildChannelMessage - ) - result_channel = Main.eval(message.expr) - process.channels[channel_id] = create_parent_channel( - process=process, - channel_id=channel_id, - result_channel=result_channel, - ) - end - - function handle_message_from_parent( - process::ParentProcess, - channel_id::UUID, - message::ChannelCloseMessage - ) - if haskey(process.channels, channel_id) && isopen(process.channels[channel_id]) - put!(process.channels[channel_id], message) - end - end - - function handle_message_from_parent( - process::ParentProcess, - channel_id::UUID, - message::ChannelTakeMessage - ) - if haskey(process.channels, channel_id) && isopen(process.channels[channel_id]) - put!(process.channels[channel_id], message) - else - respond_to_parent( - to=process, - about=channel_id, - with=ChannelCloseMessage(), - ) - end - end -end - # ╔═║ e4109311-8252-4793-87b8-eae807df7997 -function listen_for_messages_from_parent(parent_process::ParentProcess) - channels = parent_process.channels +function listen_for_messages_from_parent(process::ParentProcess) + channels = process.channels + # ?? What do these locks do?? They look cool, but are they useful actually? locks = Dict{UUID, ReentrantLock}() - for envelope in ReadMessages(parent_process.parent_to_child) + for envelope in ReadMessages(process.parent_to_child) + channel_id = envelope.channel_id + message = envelope.message + channel_lock = get!(locks, envelope.channel_id) do ReentrantLock() end + schedule(Task() do lock(channel_lock) try - handle_message_from_parent( - parent_process, - envelope.channel_id, - envelope.message, - ) + if envelope.message isa CreateChildChannelMessage + @assert !haskey(process.channels, channel_id) + + result_channel = Main.eval(message.expr) + process.channels[channel_id] = create_parent_channel( + process=process, + channel_id=channel_id, + result_channel=result_channel, + ) + else + if ( + haskey(process.channels, channel_id) && + isopen(process.channels[channel_id]) + ) + put!(process.channels[channel_id], message) + else + respond_to_parent( + to=process, + about=channel_id, + with=ChannelCloseMessage(), + ) + end + end catch error + @error "Does it error here already?" error respond_to_parent( - to=parent_process, + to=process, about=envelope.channel_id, with=ErrorMessage(CapturedException(error, catch_backtrace())) ) @@ -715,11 +725,14 @@ uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" # ╔═║ Cell order: # β•Ÿβ”€4c4b77dc-8545-4922-9897-110fa67c99f4 -# β•Ÿβ”€d3fe8144-6ba8-4dd9-b0e3-941a96422267 +# ╠═d3fe8144-6ba8-4dd9-b0e3-941a96422267 +# ╠═f99f4659-f71e-4f2e-a674-67ba69289817 +# ╠═2342d663-030f-4ed2-b1d5-5be1910b6d4c # ╠═4b42e233-1f06-49c9-8c6a-9dc21c21ffb7 # ╠═e3e16a8b-7124-4678-8fe7-12ed449e1954 # β•Ÿβ”€934d18d4-936e-4ee0-aa6e-86aa6f66774c # β•Ÿβ”€54a03cba-6d00-4632-8bd6-c60753c15ae6 +# ╠═4df77979-0c35-4139-aa5e-da43c1de3a77 # ╠═6925ffbb-fd9e-402c-a483-c78f28f892a5 # ╠═8c97d5cb-e346-4b94-b2a1-4fb5ff093bd5 # ╠═63531683-e295-4ba6-811f-63b0d384ba0f @@ -733,12 +746,12 @@ uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" # ╠═ce2acdd1-b4bb-4a8f-8346-a56dfa55f56b # ╠═87182605-f5a1-46a7-968f-bdfb9d0e08fa # ╠═b05be9c7-8cc1-47f9-8baa-db66ac83c24f -# ╠═590a7882-3d69-48b0-bb1b-f476c7f8a885 +# β•Ÿβ”€590a7882-3d69-48b0-bb1b-f476c7f8a885 # ╠═46d905fc-d41e-4c9a-a808-14710f64293a # ╠═13089c5d-f833-4fb7-b8cd-4158b1a57103 -# ╠═38ca1c5b-e27e-4458-98a7-786b2d302ba3 # ╠═e4109311-8252-4793-87b8-eae807df7997 # β•Ÿβ”€17cb5a65-2a72-4e7d-8fba-901452b2c19f +# ╠═3431051e-55ce-46c1-a0f5-364662f5c77b # ╠═85920c27-d8a6-4b5c-93b6-41daa5866f9d # ╠═af4f0720-f7f7-4d8a-bd8c-8cf5abaf10a0 # ╠═368ccd31-1681-4a4a-a103-2b9afc0813ee @@ -746,14 +759,14 @@ uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" # ╠═e787d8bd-499d-4a41-8913-62b3d9346748 # ╠═20f66652-bca0-4e47-a4b7-502cfbcb3db5 # ╠═6a7aa0ce-0ae3-4e7d-a93a-bfdebe406220 -# ╠═f99f4659-f71e-4f2e-a674-67ba69289817 -# ╠═2342d663-030f-4ed2-b1d5-5be1910b6d4c # ╠═abe897ff-6e86-40eb-a1a6-2918b6c3c5a7 # β•Ÿβ”€884e103a-8925-477d-a264-f15d02a49aa9 # ╠═b8865d63-19fa-4438-87a2-ccb531bd73a4 # ╠═e1e79669-8d0a-4b04-a7fd-469e7d8e65b1 # ╠═ed6c16ed-bf4a-40ce-9c14-a72fd77e24a1 # ╠═5fb65aec-3512-4f48-98ce-300ab9fdadfe +# β•Ÿβ”€1d63f09f-dfb2-40dd-beb5-45125cb19006 +# ╠═f81ff6f7-f230-4373-b60b-76e8a0eba929 # ╠═fc0ca4b5-e03b-4c08-b43d-913ee12269c7 # ╠═adfb2d12-07a0-4d5e-8e24-1676c07107c7 # ╠═e8256f1f-c778-4fb3-a2d3-12da0e1cb3da @@ -761,7 +774,6 @@ uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" # ╠═0ba318bc-7b4c-4d0f-a46c-2f9dca756926 # ╠═c83f2936-3768-4724-9c5c-335d7a4aae03 # ╠═441c27e4-6884-4887-9cd5-bc55d0c49760 -# ╠═f81ff6f7-f230-4373-b60b-76e8a0eba929 # β•Ÿβ”€1f868b3c-df9a-4d4d-a6a9-9a196298a3af # β•Ÿβ”€77df6f7b-ed8e-442a-9489-873fc392937c # ╠═3a62b419-eca2-4de1-89a4-fc9ad6f68372 diff --git a/src/evaluation/ChildProcessesTest.jl b/src/evaluation/ChildProcessesTest.jl index 84b3bf83f9..95029a9437 100644 --- a/src/evaluation/ChildProcessesTest.jl +++ b/src/evaluation/ChildProcessesTest.jl @@ -4,16 +4,6 @@ using Markdown using InteractiveUtils -# This Pluto notebook uses @bind for interactivity. When running this notebook outside of Pluto, the following 'mock version' of @bind gives bound variables a default value (instead of an error). -macro bind(def, element) - quote - local iv = try Base.loaded_modules[Base.PkgId(Base.UUID("6e696c72-6542-2067-7265-42206c756150"), "AbstractPlutoDingetjes")].Bonds.initial_value catch; b -> missing; end - local el = $(esc(element)) - global $(esc(def)) = Core.applicable(Base.get, el) ? Base.get(el) : iv(el) - el - end -end - # ╔═║ 7e613bd2-616a-4687-8af5-a22c7a747d97 import Serialization @@ -29,163 +19,32 @@ import PlutoLinks: @ingredients, @use_task # ╔═║ f9675ee0-e728-420b-81bd-22e57583c587 import PlutoHooks: @use_effect, @use_ref, @use_state, @use_memo -# ╔═║ ca96e0d5-0904-4ae5-89d0-c1a9187710a1 -Base.@kwdef struct PlutoProcess - process - stdin - stdout - stderr -end - -# ╔═║ b7734627-ba2a-48d8-8fd8-a5c94716da20 -function Base.show(io::IO, ::MIME"text/plain", process::PlutoProcess) - write(io, process_running(process.process) ? "Running" : "Stopped") -end - -# ╔═║ 1a2f116b-78f7-47b6-b96b-c0c74b5c5a35 -@bind reset_process_counter PlutoUI.CounterButton("Reset process!") - -# ╔═║ 8d6f46b6-e6cf-4094-a32a-f1b13dc005f6 -@bind send_counter PlutoUI.CounterButton("Send!") - -# ╔═║ 09f4b4b6-75a4-4727-9405-e4fc45c2cda7 -import Distributed - -# ╔═║ ffc98725-a88f-4dac-b387-f73cfd07510a -# @use_task([x]) do -# ChildProcesses.call(x, quote -# 1 + 1 -# end) -# end - # ╔═║ a26cc458-4578-427f-840d-71d78c5c8b01 begin ChildProcesses = @ingredients("./ChildProcesses.jl").ChildProcesses var"ChildProcesses.jl" = ChildProcesses end -# ╔═║ 730cfe5e-1541-4c08-8d4a-86d1f9e4115e -let +# ╔═║ 6ff77f91-ee9c-407c-a243-09fc7e555d73 +function with_process(fn) process = ChildProcesses.create_child_process() - - ChildProcesses.call(process, :(throw("hi"))) + try + fn(process) + finally + close(process) + end end -# ╔═║ b7ac2fea-41f4-47f3-a294-944378cb7093 -x = @use_memo([]) do - ChildProcesses.create_child_process() +# ╔═║ 730cfe5e-1541-4c08-8d4a-86d1f9e4115e +with_process() do process + ChildProcesses.call(process, :(throw("Hi"))) end -# ╔═║ 872a8651-736c-47dd-80e6-b645fb0490c9 -kill(x, Base.SIGINT) - -# ╔═║ c1038e6a-c4e9-4b79-b3f9-35a9f64b0f64 -task = @use_task([x]) do - ChildProcesses.call(x, quote - while true; end - end) +# ╔═║ 6ab96c70-ad5d-4614-9a77-2d44d1085567 +with_process() do process + ChildProcesses.call(process, :(1 + 1)) end -# ╔═║ 75fc343a-7904-45f8-be09-a542255ba673 -istaskdone(task) ? fetch(task) : nothing - -# ╔═║ 281b4aab-307a-4d90-9dfb-f422b9567736 -# process_output = let -# # error("Nope") - -# my_stderr = @use_memo([reset_process_counter, ChildProcesses]) do -# Pipe() -# end -# output, set_output = @use_state("") - -# process = @use_memo([my_stderr]) do -# ChildProcesses.create_child_process( -# custom_stderr=my_stderr, -# exeflags=["--color=yes", "--threads=4"], -# ) -# end -# @use_effect([process]) do -# return () -> begin -# kill(process) -# end -# end - -# # So we re-run the whole thing when the process exists -# _, refresh_state = @use_state(nothing) -# @use_task([]) do -# if process_running(process) -# wait(process) -# refresh_state(nothing) -# end -# end - -# @use_task([]) do -# while process_running(process) && !eof(my_stderr) -# new_output = String(readavailable(my_stderr)) -# set_output((output) -> begin -# output * new_output -# end) -# end -# end - -# pluto_process = @use_memo([process, my_stderr]) do -# PlutoProcess( -# process=process, -# stderr=my_stderr, -# stdin=process.process.in, -# stdout=process.process.out, -# ) -# end - -# PlutoUI.with_terminal() do -# print(output) -# pluto_process -# end -# end - -# ╔═║ e5edaa4d-74ff-4f6e-a045-71fd5494dd79 -# @use_memo([spawned_process]) do -# try -# ChildProcesses.create_channel(spawned_process, quote -# Channel() do ch -# for i in 1:2000 -# put!(ch, i) -# end -# end -# end) do ch -# for i in 1:10 -# x = take!(ch) -# end -# end -# catch error -# @error "UHHH" error stacktrace(catch_backtrace()) -# (error, stacktrace(catch_backtrace())) -# end -# end - -# ╔═║ 5fb236c3-b67d-47ee-8644-84bd51e577b1 -# @task_result([spawned_process]) do -# ChildProcesses.call(spawned_process, quote -# 1 + 1 -# end) -# end - -# ╔═║ 8bd24b7b-4837-46b7-a6e9-b674630c2f56 -# @use_memo([spawned_process]) do -# for i in 1:10 -# ChildProcesses.call(spawned_process, quote -# 1 + 1 -# end) -# end -# end - -# ╔═║ e33a9b31-722e-425e-be5e-b33517bec8e3 -# @use_memo([spawned_process]) do -# BenchmarkTools.@benchmark ChildProcesses.call(spawned_process, quote -# 1 + 1 -# end) -# end - # ╔═║ de602aaa-704c-45c4-8b7b-fc58e41236ce begin struct FakeProcess <: Base.AbstractPipe @@ -197,78 +56,12 @@ begin eval(:(Base.pipe_reader(process::FakeProcess) = process.out)) end -# ╔═║ abf694f4-c5c4-4a4f-b30f-62e358149195 -# @task_result() do -# Base.readbytes!(PipeBuffer(), UInt8[], 10) -# end - -# ╔═║ ec4a5558-28dd-47c1-b015-8799d9cb8800 -# function Base.eof(buffer::IOBuffer) -# while buffer.readable -# if bytesavailable(buffer) !== 0 -# return false -# end -# # eval(:(@info "OOPS")) -# sleep(1) -# yield() -# end -# return true -# end - -# ╔═║ ce384249-52c5-47d8-9c93-18837432b625 -# let -# parent_to_child = PipeBuffer() -# child_to_parent = PipeBuffer() -# child_process = @use_memo([ChildProcesses.ChildProcess, FakeProcess]) do -# process = FakeProcess(child_to_parent, parent_to_child) -# child_process = ChildProcesses.ChildProcess(process=process) -# end -# parent_process = @use_memo([ChildProcesses.ParentProcess]) do -# ChildProcesses.ParentProcess( -# parent_to_child=parent_to_child, -# child_to_parent=child_to_parent, -# ) -# end - -# @info "#1" -# @use_task([ChildProcesses.start_from_child_loop, child_process]) do -# ChildProcesses.start_from_child_loop(child_process) -# end -# @info "#2" -# @use_task([ChildProcesses.listen_for_messages_from_parent, parent_process]) do -# ChildProcesses.listen_for_messages_from_parent(parent_process) -# end -# @info "#3" - -# result, set_result = @use_state(Pending()) - -# @use_task([ChildProcesses.call, child_process]) do -# try -# result = ChildProcesses.call(child_process, quote -# 1 + 1 -# end) - -# set_result(Result(result)) -# catch error -# set_result(Failure(error)) -# end -# end - -# result -# end - -# ╔═║ 4be7e097-72a3-4590-bcfb-a7dacb78159c -spawned_process = process_output.value.process; - # ╔═║ 49fdc8a3-0e1a-42f0-acc4-b823eec91d31 md"---" # ╔═║ 95c5a5bc-db23-4ad3-8ae8-81bc8f0edfd4 import BenchmarkTools -# ╔═║ 4541f52f-1217-4dcd-b44c-042c7ca246bd -1 + 1 - # ╔═║ ced9d1e9-7075-4ff2-8ca2-6a349f2a69c4 let stream = PipeBuffer() @@ -283,6 +76,19 @@ let end end +# ╔═║ 4baac7f2-60fe-4a6f-8612-2acf80c43ef3 +let + process = ChildProcesses.create_child_process() + + benchmark = BenchmarkTools.@benchmark begin + ChildProcesses.call($process, :(1 + 1)) + end + + kill(process) + + benchmark +end + # ╔═║ be18d157-6b55-4eaa-99fe-c398e992a9fa md"### @task_result" @@ -299,7 +105,6 @@ struct Failure error end PLUTO_PROJECT_TOML_CONTENTS = """ [deps] BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" -Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" PlutoHooks = "0ff47ea0-7a50-410d-8455-4348d5de0774" PlutoLinks = "0ff47ea0-7a50-410d-8455-4348d5de0420" PlutoUI = "7f904dfe-b85e-4ff6-b463-dae2292396a8" @@ -330,10 +135,6 @@ version = "1.2.0" deps = ["Printf"] uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" -[[Distributed]] -deps = ["Random", "Serialization", "Sockets"] -uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" - [[FileWatching]] uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" @@ -428,9 +229,6 @@ uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" [[Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" -[[Sockets]] -uuid = "6462fe0b-24de-5631-8697-dd941f90decc" - [[SparseArrays]] deps = ["LinearAlgebra", "Random"] uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" @@ -457,32 +255,15 @@ uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" # ╠═f7d14367-27d7-41a5-9f6a-79cf5e721a7d # ╠═fe669218-18c3-46c7-80e8-7b1ab6fa77d2 # ╠═f9675ee0-e728-420b-81bd-22e57583c587 -# β•Ÿβ”€ca96e0d5-0904-4ae5-89d0-c1a9187710a1 -# ╠═b7734627-ba2a-48d8-8fd8-a5c94716da20 -# β•Ÿβ”€1a2f116b-78f7-47b6-b96b-c0c74b5c5a35 -# β•Ÿβ”€8d6f46b6-e6cf-4094-a32a-f1b13dc005f6 -# ╠═09f4b4b6-75a4-4727-9405-e4fc45c2cda7 # ╠═730cfe5e-1541-4c08-8d4a-86d1f9e4115e -# ╠═b7ac2fea-41f4-47f3-a294-944378cb7093 -# ╠═c1038e6a-c4e9-4b79-b3f9-35a9f64b0f64 -# ╠═75fc343a-7904-45f8-be09-a542255ba673 -# ╠═ffc98725-a88f-4dac-b387-f73cfd07510a -# ╠═872a8651-736c-47dd-80e6-b645fb0490c9 +# ╠═6ff77f91-ee9c-407c-a243-09fc7e555d73 +# ╠═6ab96c70-ad5d-4614-9a77-2d44d1085567 # ╠═a26cc458-4578-427f-840d-71d78c5c8b01 -# ╠═281b4aab-307a-4d90-9dfb-f422b9567736 -# ╠═e5edaa4d-74ff-4f6e-a045-71fd5494dd79 -# ╠═5fb236c3-b67d-47ee-8644-84bd51e577b1 -# ╠═8bd24b7b-4837-46b7-a6e9-b674630c2f56 -# ╠═e33a9b31-722e-425e-be5e-b33517bec8e3 # ╠═de602aaa-704c-45c4-8b7b-fc58e41236ce -# ╠═abf694f4-c5c4-4a4f-b30f-62e358149195 -# ╠═ec4a5558-28dd-47c1-b015-8799d9cb8800 -# ╠═ce384249-52c5-47d8-9c93-18837432b625 -# ╠═4be7e097-72a3-4590-bcfb-a7dacb78159c # β•Ÿβ”€49fdc8a3-0e1a-42f0-acc4-b823eec91d31 # ╠═95c5a5bc-db23-4ad3-8ae8-81bc8f0edfd4 -# ╠═4541f52f-1217-4dcd-b44c-042c7ca246bd # ╠═ced9d1e9-7075-4ff2-8ca2-6a349f2a69c4 +# ╠═4baac7f2-60fe-4a6f-8612-2acf80c43ef3 # β•Ÿβ”€be18d157-6b55-4eaa-99fe-c398e992a9fa # ╠═12578b59-0161-4e72-afef-825166a62121 # ╠═34d90560-5a3e-4c7f-8126-35e1a6153aa1 From 917ae62fbc875a537133069ef4c2b2926c9521d9 Mon Sep 17 00:00:00 2001 From: Michiel Dral Date: Tue, 25 Jan 2022 23:00:27 +0100 Subject: [PATCH 26/28] tryin trying trying --- sample/ChildProcesses_benchmark.jl | 166 +++++++++++++++++++++++ src/evaluation/ChildProcessesNotebook.jl | 12 +- src/evaluation/Run.jl | 3 + src/evaluation/WorkspaceManager.jl | 18 +-- 4 files changed, 184 insertions(+), 15 deletions(-) create mode 100644 sample/ChildProcesses_benchmark.jl diff --git a/sample/ChildProcesses_benchmark.jl b/sample/ChildProcesses_benchmark.jl new file mode 100644 index 0000000000..f3938d453a --- /dev/null +++ b/sample/ChildProcesses_benchmark.jl @@ -0,0 +1,166 @@ +### A Pluto.jl notebook ### +# v0.17.7 + +using Markdown +using InteractiveUtils + +# ╔═║ 03cbe047-96c6-456f-8adb-8d80390a1742 +md""" +# ChildProcesses Benchmark + +To run this you must be running ChildProcesses enabled Pluto (so you can `Distributed`) +""" + +# ╔═║ 8e626d50-b8a1-4353-9193-9297d91d6352 +import Distributed + +# ╔═║ 808241a9-c7ea-4ca0-9a8b-74658c50a48a +import BenchmarkTools + +# ╔═║ 9b582916-0fd7-4024-9877-1014417c545e +# Act like you a package, quick! +ChildProcesses = @eval Main module ChildProcesses2 + include("../src/evaluation/ChildProcessesNotebook.jl") +end + +# ╔═║ f2e2a5de-57dd-4ae7-9860-efa693ef786a + + +# ╔═║ 7a3e1cc4-b188-42f2-96c7-931f20ccfa34 +expr_to_test = quote + $(fill(1, 1000)) +end + +# ╔═║ 1326fab3-c8ef-4b0d-a17e-36cbcb7a5275 +md"## Distributed.remotecall_eval" + +# ╔═║ 3af58ffa-1d51-42b8-b70e-83a160011ede +let + (pid,) = Distributed.addprocs(1) + + benchmark = BenchmarkTools.@benchmark begin + Distributed.remotecall_eval(Main, [$pid], $expr_to_test) + end + + Distributed.rmprocs([pid]) + benchmark +end + +# ╔═║ 0a632e40-73b2-4816-b846-089af80f76cc +md"## ChildProcesses.call" + +# ╔═║ 5c8e1389-94c3-4abc-a9fd-698e7013bcf6 +let + process = ChildProcesses.create_child_process() + + benchmark = BenchmarkTools.@benchmark begin + ChildProcesses.call($process, $expr_to_test) + end + + kill(process) + benchmark +end + +# ╔═║ 00000000-0000-0000-0000-000000000001 +PLUTO_PROJECT_TOML_CONTENTS = """ +[deps] +BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[compat] +BenchmarkTools = "~1.2.2" +""" + +# ╔═║ 00000000-0000-0000-0000-000000000002 +PLUTO_MANIFEST_TOML_CONTENTS = """ +# This file is machine-generated - editing it directly is not advised + +[[BenchmarkTools]] +deps = ["JSON", "Logging", "Printf", "Profile", "Statistics", "UUIDs"] +git-tree-sha1 = "940001114a0147b6e4d10624276d56d531dd9b49" +uuid = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +version = "1.2.2" + +[[Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[JSON]] +deps = ["Dates", "Mmap", "Parsers", "Unicode"] +git-tree-sha1 = "8076680b162ada2a031f707ac7b4953e30667a37" +uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +version = "0.21.2" + +[[Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[LinearAlgebra]] +deps = ["Libdl"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + +[[Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" + +[[Parsers]] +deps = ["Dates"] +git-tree-sha1 = "92f91ba9e5941fc781fecf5494ac1da87bdac775" +uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" +version = "2.2.0" + +[[Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[Profile]] +deps = ["Printf"] +uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" + +[[Random]] +deps = ["Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" + +[[Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[SparseArrays]] +deps = ["LinearAlgebra", "Random"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + +[[Statistics]] +deps = ["LinearAlgebra", "SparseArrays"] +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" + +[[UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" +""" + +# ╔═║ Cell order: +# ╠═03cbe047-96c6-456f-8adb-8d80390a1742 +# ╠═8e626d50-b8a1-4353-9193-9297d91d6352 +# ╠═808241a9-c7ea-4ca0-9a8b-74658c50a48a +# ╠═9b582916-0fd7-4024-9877-1014417c545e +# β•Ÿβ”€f2e2a5de-57dd-4ae7-9860-efa693ef786a +# ╠═7a3e1cc4-b188-42f2-96c7-931f20ccfa34 +# β•Ÿβ”€1326fab3-c8ef-4b0d-a17e-36cbcb7a5275 +# ╠═3af58ffa-1d51-42b8-b70e-83a160011ede +# β•Ÿβ”€0a632e40-73b2-4816-b846-089af80f76cc +# ╠═5c8e1389-94c3-4abc-a9fd-698e7013bcf6 +# β•Ÿβ”€00000000-0000-0000-0000-000000000001 +# β•Ÿβ”€00000000-0000-0000-0000-000000000002 diff --git a/src/evaluation/ChildProcessesNotebook.jl b/src/evaluation/ChildProcessesNotebook.jl index 4e6b00afe5..e51d235974 100644 --- a/src/evaluation/ChildProcessesNotebook.jl +++ b/src/evaluation/ChildProcessesNotebook.jl @@ -91,6 +91,11 @@ md"## Message types" # ╔═║ fc0ca4b5-e03b-4c08-b43d-913ee12269c7 abstract type Message end +# ╔═║ 3fd22ac5-e1c9-42a2-8ae1-7cc4e154764a +struct JustExecuteThis <: Message + expr +end + # ╔═║ 3431051e-55ce-46c1-a0f5-364662f5c77b MessageChannel = AbstractChannel{Message} @@ -725,9 +730,9 @@ uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" # ╔═║ Cell order: # β•Ÿβ”€4c4b77dc-8545-4922-9897-110fa67c99f4 -# ╠═d3fe8144-6ba8-4dd9-b0e3-941a96422267 -# ╠═f99f4659-f71e-4f2e-a674-67ba69289817 -# ╠═2342d663-030f-4ed2-b1d5-5be1910b6d4c +# β•Ÿβ”€d3fe8144-6ba8-4dd9-b0e3-941a96422267 +# β•Ÿβ”€f99f4659-f71e-4f2e-a674-67ba69289817 +# β•Ÿβ”€2342d663-030f-4ed2-b1d5-5be1910b6d4c # ╠═4b42e233-1f06-49c9-8c6a-9dc21c21ffb7 # ╠═e3e16a8b-7124-4678-8fe7-12ed449e1954 # β•Ÿβ”€934d18d4-936e-4ee0-aa6e-86aa6f66774c @@ -749,6 +754,7 @@ uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" # β•Ÿβ”€590a7882-3d69-48b0-bb1b-f476c7f8a885 # ╠═46d905fc-d41e-4c9a-a808-14710f64293a # ╠═13089c5d-f833-4fb7-b8cd-4158b1a57103 +# ╠═3fd22ac5-e1c9-42a2-8ae1-7cc4e154764a # ╠═e4109311-8252-4793-87b8-eae807df7997 # β•Ÿβ”€17cb5a65-2a72-4e7d-8fba-901452b2c19f # ╠═3431051e-55ce-46c1-a0f5-364662f5c77b diff --git a/src/evaluation/Run.jl b/src/evaluation/Run.jl index d0dc39a2f3..3f27d7a7b3 100644 --- a/src/evaluation/Run.jl +++ b/src/evaluation/Run.jl @@ -409,6 +409,9 @@ function update_save_run!(session::ServerSession, notebook::Notebook, cells::Arr update_dependency_cache!(notebook) save && save_notebook(session, notebook) + # TODO REMOVE THIS BEFORE MERGING!!! + prerender_text = false + # _assume `prerender_text == false` if you want to skip some details_ to_run_online = if !prerender_text cells diff --git a/src/evaluation/WorkspaceManager.jl b/src/evaluation/WorkspaceManager.jl index df664f1b2d..818057f98e 100644 --- a/src/evaluation/WorkspaceManager.jl +++ b/src/evaluation/WorkspaceManager.jl @@ -114,7 +114,7 @@ function use_nbpkg_environment((session, notebook)::SN, workspace=nothing) end end -function start_relaying_self_updates((session, notebook)::SN, run_channel::Distributed.RemoteChannel) +function start_relaying_self_updates((session, notebook)::SN, run_channel::ChildProcesses.ChildChannel) while true try next_run_uuid = take!(run_channel) @@ -192,21 +192,15 @@ end function possible_bond_values(session_notebook::SN, n::Symbol; get_length::Bool=false) workspace = get_workspace(session_notebook) - pid = workspace.pid - - Distributed.remotecall_eval(Main, pid, quote + ChildProcesses.call(workspace.process, quote PlutoRunner.possible_bond_values($(QuoteNode(n)); get_length=$(get_length)) end) end -function create_emptyworkspacemodule(pid::Integer)::Symbol +function create_emptyworkspacemodule(process::ChildProcesses.ChildProcess)::Symbol ChildProcesses.call(process, :(PlutoRunner.increment_current_module())) end -const Distributed_expr = :( - Base.loaded_modules[Base.PkgId(Base.UUID("8ba89e20-285c-5b6f-9357-94700520ee1b"), "Distributed")] -) - # NOTE: this function only start a worker process using given # compiler options, it does not resolve paths for notebooks # compiler configurations passed to it should be resolved before this @@ -217,9 +211,7 @@ function create_workspaceprocess(;compiler_options=CompilerOptions()) exeflags = _convert_to_flags(compiler_options) process = ChildProcesses.create_child_process(exeflags=exeflags) - for expr in process_preamble - ChildProcesses.call_without_fetch(process, expr) - end + ChildProcesses.call_without_fetch(process, process_preamble) # so that we NEVER break the workspace with an interrupt πŸ€• @async ChildProcesses.call_without_fetch(process, quote @@ -280,6 +272,7 @@ function distributed_exception_result(ex::ChildProcesses.ChildProcessException, process_exited=false, runtime=nothing, published_objects=Dict{String,Any}(), + has_pluto_hook_features=false, ) end @@ -291,6 +284,7 @@ function distributed_exception_result(ex::ChildProcesses.ProcessExitedException, process_exited=true && !workspace.discarded, # don't report a process exit if the workspace was discarded on purpose runtime=nothing, published_objects=Dict{String,Any}(), + has_pluto_hook_features=false, ) end From a158e6b8c2fe0c9fa1a6feeb6c52a331a14cc827 Mon Sep 17 00:00:00 2001 From: Michiel Dral Date: Tue, 25 Jan 2022 23:22:37 +0100 Subject: [PATCH 27/28] Was hoping this would give me more insight but mehh --- sample/ChildProcesses_benchmark.jl | 48 +++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/sample/ChildProcesses_benchmark.jl b/sample/ChildProcesses_benchmark.jl index f3938d453a..1895fa94db 100644 --- a/sample/ChildProcesses_benchmark.jl +++ b/sample/ChildProcesses_benchmark.jl @@ -8,7 +8,7 @@ using InteractiveUtils md""" # ChildProcesses Benchmark -To run this you must be running ChildProcesses enabled Pluto (so you can `Distributed`) +To run this you must be running ChildProcesses enabled Pluto (so you can `Distributed`). """ # ╔═║ 8e626d50-b8a1-4353-9193-9297d91d6352 @@ -28,14 +28,14 @@ end # ╔═║ 7a3e1cc4-b188-42f2-96c7-931f20ccfa34 expr_to_test = quote - $(fill(1, 1000)) + $(fill(1, (1000, 1000))) end # ╔═║ 1326fab3-c8ef-4b0d-a17e-36cbcb7a5275 md"## Distributed.remotecall_eval" -# ╔═║ 3af58ffa-1d51-42b8-b70e-83a160011ede -let +# ╔═║ 28be2c7a-e2b2-4666-a367-062e3f72ec77 +function distributed_benchmark(expr_to_test) (pid,) = Distributed.addprocs(1) benchmark = BenchmarkTools.@benchmark begin @@ -46,21 +46,45 @@ let benchmark end +# ╔═║ 11635702-a887-4548-a6b8-55a1c7b08ee7 +distributed_benchmark(quote $(1) end) + +# ╔═║ d77fb79a-7105-4321-9151-84d9367c12dc +distributed_benchmark(quote $(fill(1, (1000))) end) + +# ╔═║ 3af58ffa-1d51-42b8-b70e-83a160011ede +distributed_benchmark(quote $(fill(1, (1000, 1000))) end) + +# ╔═║ 7bb5fa42-5f7d-4e83-a55c-ed435f4d4746 +distributed_benchmark(quote $(fill(1, (1000, 1000, 10))) end) + # ╔═║ 0a632e40-73b2-4816-b846-089af80f76cc md"## ChildProcesses.call" -# ╔═║ 5c8e1389-94c3-4abc-a9fd-698e7013bcf6 -let +# ╔═║ ba51da9b-44a0-41e6-bd37-b05eed451906 +function childprocess_benchmark(expr_to_test) process = ChildProcesses.create_child_process() benchmark = BenchmarkTools.@benchmark begin - ChildProcesses.call($process, $expr_to_test) + $ChildProcesses.call($process, $expr_to_test) end kill(process) benchmark end +# ╔═║ f303a2eb-d4ff-438d-b01a-45bc6a62bf87 +childprocess_benchmark(quote $(1) end) + +# ╔═║ 8f6bbed5-e259-4841-a9bc-113df6f13676 +childprocess_benchmark(quote $(fill(1, (1000))) end) + +# ╔═║ 5c8e1389-94c3-4abc-a9fd-698e7013bcf6 +childprocess_benchmark(quote $(fill(1, (1000, 1000))) end) + +# ╔═║ ef907db1-721d-4695-8752-8fe44db1a0a5 +childprocess_benchmark(quote $(fill(1, (1000, 1000, 10))) end) + # ╔═║ 00000000-0000-0000-0000-000000000001 PLUTO_PROJECT_TOML_CONTENTS = """ [deps] @@ -152,15 +176,23 @@ uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" """ # ╔═║ Cell order: -# ╠═03cbe047-96c6-456f-8adb-8d80390a1742 +# β•Ÿβ”€03cbe047-96c6-456f-8adb-8d80390a1742 # ╠═8e626d50-b8a1-4353-9193-9297d91d6352 # ╠═808241a9-c7ea-4ca0-9a8b-74658c50a48a # ╠═9b582916-0fd7-4024-9877-1014417c545e # β•Ÿβ”€f2e2a5de-57dd-4ae7-9860-efa693ef786a # ╠═7a3e1cc4-b188-42f2-96c7-931f20ccfa34 # β•Ÿβ”€1326fab3-c8ef-4b0d-a17e-36cbcb7a5275 +# β•Ÿβ”€28be2c7a-e2b2-4666-a367-062e3f72ec77 +# ╠═11635702-a887-4548-a6b8-55a1c7b08ee7 +# ╠═d77fb79a-7105-4321-9151-84d9367c12dc # ╠═3af58ffa-1d51-42b8-b70e-83a160011ede +# ╠═7bb5fa42-5f7d-4e83-a55c-ed435f4d4746 # β•Ÿβ”€0a632e40-73b2-4816-b846-089af80f76cc +# β•Ÿβ”€ba51da9b-44a0-41e6-bd37-b05eed451906 +# ╠═f303a2eb-d4ff-438d-b01a-45bc6a62bf87 +# ╠═8f6bbed5-e259-4841-a9bc-113df6f13676 # ╠═5c8e1389-94c3-4abc-a9fd-698e7013bcf6 +# ╠═ef907db1-721d-4695-8752-8fe44db1a0a5 # β•Ÿβ”€00000000-0000-0000-0000-000000000001 # β•Ÿβ”€00000000-0000-0000-0000-000000000002 From 945cef2b4207d5b504f1bc346cd53da2fad0546a Mon Sep 17 00:00:00 2001 From: Fons van der Plas Date: Wed, 26 Jan 2022 18:59:58 +0100 Subject: [PATCH 28/28] Update Project.toml --- Project.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Project.toml b/Project.toml index 6bf870218f..6e455f5768 100644 --- a/Project.toml +++ b/Project.toml @@ -18,6 +18,7 @@ Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a" MsgPack = "99f44e22-a591-53d1-9472-aa23ef4bd671" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" REPL = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" +Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"