From 2d74304bc89c2ff76b71c6d07dee1de484619254 Mon Sep 17 00:00:00 2001 From: Jun Tian Date: Tue, 31 Aug 2021 15:54:01 +0800 Subject: [PATCH 1/4] the minimal implementation --- Project.toml | 2 + README.md | 151 +++++------ src/Oolong.jl | 4 +- src/core.jl | 569 ++++++++++------------------------------ src/parameter_server.jl | 21 -- src/scheduler.jl | 15 -- src/serve.jl | 114 -------- 7 files changed, 205 insertions(+), 671 deletions(-) delete mode 100644 src/parameter_server.jl delete mode 100644 src/scheduler.jl delete mode 100644 src/serve.jl diff --git a/Project.toml b/Project.toml index 8c8270e..b4f2edd 100644 --- a/Project.toml +++ b/Project.toml @@ -7,6 +7,8 @@ version = "0.1.0" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" +URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" +UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [compat] julia = "1" diff --git a/README.md b/README.md index 07e3de9..85a4248 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Oolong.jl -*An actor framework for [ReinforcementLearning.jl](https://github.com/JuliaReinforcementLearning/ReinforcementLearning.jl)* +*An actor framework for [~~ReinforcementLearning.jl~~](https://github.com/JuliaReinforcementLearning/ReinforcementLearning.jl) distributed computing in Julia.* > “是非成败转头空” —— [《临江仙》](https://www.vincentpoon.com/the-immortals-by-the-river-----------------.html) > [杨慎](https://zh.wikipedia.org/zh-hans/%E6%9D%A8%E6%85%8E) @@ -12,118 +12,87 @@ > > (Translated by [Xu Yuanchong](https://en.wikipedia.org/wiki/Xu_Yuanchong)) -## Roadmap - -- [x] Figure out a set of simple primitives for running distributed - applications. -- [ ] Apply this package to some typical RL algorithms: - - [x] Parameter server - - [x] Batch serving - - [ ] Add macro to expose a http endpoint - - [ ] A3C - - [ ] D4PG - - [ ] AlphaZero - - [ ] Deep CFR - - [ ] NFSP - - [ ] Evolution algorithms -- [ ] Resource management across nodes -- [ ] State persistence and fault tolerance -- [ ] Configurable logging and dashboard - - [LokiLogger.jl](https://github.com/fredrikekre/LokiLogger.jl) - - [Stipple.jl](https://github.com/GenieFramework/Stipple.jl) - -## Get Started -⚠ *This package is still under rapid development and is not registered yet.* +## Features -First install this package: +- Non-invasive + Users can easily extend existing packages to apply them in a cluster. -```julia -pkg> activate --temp - -pkg> add https://github.com/JuliaReinforcementLearning/Oolong.jl -``` +- Simple API + -`Oolong.jl` adopts the [actor model](https://en.wikipedia.org/wiki/Actor_model) to -parallelize your existing code. One of the core APIs defined in this package is -the `@actor` macro. - -```julia -using Oolong - -A = @actor () -> @info "Hello World" -``` - -By putting the `@actor` macro before arbitrary callable object, we defined an -**actor**. And we can call it as usual: - -```julia -A(); -``` +## Roadmap -You'll see something like this on your screen: +- Stage 1 + - [ ] Stabilize API + - [ ] `@pot`, define a container over any callable object. + - [ ] `-->`, `<--`, define a streaming pipeline. + - [ ] Example usages +- Stage 2 + - [ ] Auto-scaling. Allow workers join/exit. + - [ ] Custom cluster manager + - [ ] Dashboard + - [ ] [grafana](https://grafana.com/) + - [ ] Custom Logger + - [LokiLogger.jl](https://github.com/fredrikekre/LokiLogger.jl) + - [Stipple.jl](https://github.com/GenieFramework/Stipple.jl) +- Stage 3 + - [ ] Drop out Distributed.jl? + - [ ] K8S + +## Design ``` -Info:[2021-06-30 22:59:51](@/user/#1)Hello World + +--------+ + | Flavor | + +--------+ + | + V +-------------+ + +---+---+ | Pot | + | PotID |<===>| | + +---+---+ | PotID | + | | () -> Tea | + | +-------------+ + +-------|-------------------------+ + | V boiled somewhere | + | +----+----+ | + | | Channel | | + | +----+----+ | + | | | + | V +-----------+ | + | +--+--+ | PotState | | + | | Tea |<===>| | | + | +--+--+ | Children | | + | | +-----------+ | + | V | + | +----+----+ | + | | Future | | + | +---------+ | + +---------------------------------+ ``` -Next, let's make sure anonymous functions with positional and keyword arguments -can also work as expected: +A `Pot` is mainly a container of an arbitrary object (`tea`) which is instantiated by calling a parameterless function. Whenever a `Pot` receives a `flavor` through the `channel`, the water in the `Pot` is *boiled* first (a `task` to process `tea` and `flavor` is created) if it is cool (the previous `task` was exited by accident or on demand). Users can define how `tea` and `flavor` are processed through multiple dispatch on `process(tea, flavor)`. In some `task`s, users may create many other `Pot`s whose references (`PotID`) are stored in `Children`. A `PotID` is simply a path used to locate a `Pot`. -```julia -A = @actor (msg;suffix="!") -> @info "Hello " * msg * suffix -A("World";suffix="!!!") -# Info:[2021-06-30 23:00:38](@/user/#5)Hello World!!! -``` - -For some functions, we are more interested in the returned value. - -```julia -A = @actor msg -> "Hello " * msg -res = A("World") -``` - -Well, different from the general function call, a result similar to `Future` is -returned instead of the real value. We can then fetch the result with the -following syntax: +## Get Started -```julia -res[] -# "Hello World" -``` +⚠ *This package is still under rapid development and is not registered yet.* -To maintain the internal states across different calls, we can also apply `@actor` -to a customized structure: +First install this package: ```julia -Base.@kwdef mutable struct Counter - n::Int = 0 -end - -(c::Counter)() = c.n += 1 - -A = @actor Counter() - -for _ in 1:10 - A() -end - -n = A.n +pkg> activate --temp -n[] -# 10 +pkg> add https://github.com/JuliaReinforcementLearning/Oolong.jl ``` -Note that similar to function call, the return of `A.n` is also a `Future` like object. - -### Tips -- Be careful with `self()` +### FAQ ## Acknowledgement This package is mainly inspired by the following packages: -- [Actors.jl](https://github.com/JuliaActors/Actors.jl) - [Proto.Actor](https://proto.actor/) - [Ray](https://ray.io/) +- [Orleans](https://github.com/dotnet//orleans) +- [Actors.jl](https://github.com/JuliaActors/Actors.jl) diff --git a/src/Oolong.jl b/src/Oolong.jl index 1fb3a9c..8b1c36b 100644 --- a/src/Oolong.jl +++ b/src/Oolong.jl @@ -4,11 +4,9 @@ const OL = Oolong export OL include("core.jl") -include("parameter_server.jl") -include("serve.jl") function __init__() - init() + start() end end diff --git a/src/core.jl b/src/core.jl index 946e9fd..067f5a7 100644 --- a/src/core.jl +++ b/src/core.jl @@ -1,507 +1,222 @@ -export @actor +export @P_str, @pot -using Base.Threads using Distributed -using Dates -using Logging +using UUIDs:uuid4 -const ACTOR_KEY = "OOLONG" +const KEY = :OOLONG ##### -# System Messages +# Pot Definition ##### -abstract type AbstractSysMsg end - -struct SuccessMsg{M} <: AbstractSysMsg - msg::M +struct PotID + path::Tuple{Vararg{Symbol}} end -struct FailureMsg{R} <: AbstractSysMsg - reason::R +struct Pot + pid::PotID + tea_bag::Any end -Base.getindex(msg::FailureMsg) = msg.reason - -Base.@kwdef struct StartMsg{F} <: AbstractSysMsg - info::F = nothing +function Pot( + tea_bag; + name=string(uuid4()) +) + pid = PotID(name) + Pot(pid, tea_bag) end -struct StopMsg{R} <: AbstractSysMsg - reason::R +macro pot(tea, kw...) + tea_bag = esc(:(() -> ($(tea)))) + xs = [esc(x) for x in kw] + quote + p = Pot($tea_bag; $(xs...)) + register(p) + p.pid + end end -struct RestartMsg <: AbstractSysMsg end -struct PreRestartMsg <: AbstractSysMsg end -struct PostRestartMsg <: AbstractSysMsg end -struct ResumeMsg <: AbstractSysMsg end +""" + P"[/]your/pot/path" -struct StatMsg <: AbstractSysMsg end +The path can be either relative or absolute path. If a relative path is provided, it will be resolved to an absolute path based on the current context. -struct FutureWrapper - f::Future - FutureWrapper(args...) = new(Future(args...)) +!!! note + We don't validate the path for you during construction. A [`PotNotRegisteredError`](@ref) will be thrown when you try to send messages to an unregistered path. +""" +macro P_str(s) + PotID(s) end -function Base.getindex(f::FutureWrapper) - res = getindex(f.f) - if res isa SuccessMsg - res.msg - elseif res isa FailureMsg{<:Exception} - throw(res.reason) +function Base.show(io::IO, p::PotID) + if isempty(p.path) + print(io, '/') else - res + for x in p.path + print(io, '/') + print(io, x) + end end end -Base.put!(f::FutureWrapper, x) = put!(f.f, x) - -##### -# Mailbox -##### - -struct Mailbox - ch::RemoteChannel -end - -const DEFAULT_MAILBOX_SIZE = typemax(Int) - -Mailbox(; size=DEFAULT_MAILBOX_SIZE, pid=myid()) = Mailbox(RemoteChannel(() -> Channel(size), pid)) - -Base.take!(m::Mailbox) = take!(getfield(m, :ch)) - -Base.put!(m::Mailbox, msg) = put!(getfield(m, :ch), msg) - -whereis(m::Mailbox) = getfield(m, :ch).where - -#= -Actor Hierarchy - -NOBODY - └── ROOT - ├── LOGGER - ├── SCHEDULER - | ├── WORKER_1 - | ├── ... - | └── WORKER_N - └── USER - ├── foo - └── bar - └── baz -=# - -struct NoBody end -const NOBODY = NoBody() - -struct RootActor end -const ROOT_ACTOR = RootActor() -const ROOT = Ref{Mailbox}() - -struct SchedulerActor end -const SCHEDULER_ACTOR = SchedulerActor() -const SCHEDULER = Ref{Mailbox}() - -struct SchedulerWorker -end - -struct StagingActor end - -struct UserActor end -const USER_ACTOR = UserActor() -const USER = Ref{Mailbox}() - -struct LoggerActor end -const LOGGER_ACTOR = LoggerActor() -const LOGGER = Ref{Mailbox}() - -##### -# RemoteLogger -##### - -struct RemoteLogger <: AbstractLogger - mailbox - min_level -end - -struct LogMsg - args - kwargs +function PotID(s::String) + if length(s) > 0 + if s[1] == '/' + PotID(Tuple(Symbol(x) for x in split(s, '/';keepempty=false))) + else + PotID((self().path..., (Symbol(x) for x in split(s, '/';keepempty=false))...)) + end + else + PotID(()) + end end -const DATE_FORMAT = "yyyy-mm-dd HH:MM:SS" +const ROOT = P"/" +const USER = P"/user" +const SCHEDULER = P"/scheduler" -function Logging.handle_message(logger::RemoteLogger, args...; kwargs...) - kwargs = merge(kwargs.data,( - datetime="$(Dates.format(now(), DATE_FORMAT))", - path=_self().path - )) - logger.mailbox[LogMsg(args, kwargs)] +self() = try + task_local_storage(KEY) +catch + USER end -Logging.shouldlog(::RemoteLogger, args...) = true -Logging.min_enabled_level(L::RemoteLogger) = L.min_level +Base.parent() = parent(self()) +Base.parent(p::PotID) = PotID(p.path[1:end-1]) ##### -# Actor +# Pot Scheduling ##### -Base.@kwdef struct Actor - path::String - thunk::Any - owner::Union{NoBody,Mailbox} - children::Dict{String,Mailbox} - taskref::Ref{Task} - mailbox::Ref{Mailbox} - mailbox_size::Int -end - -Base.nameof(a::Actor) = basename(a.path) +"local cache to reduce remote call, we may use redis like db later" +const POT_LINK_CACHE = Dict{PotID, RemoteChannel{Channel{Any}}}() +const POT_REGISTRY_CACHE = Dict{PotID, Pot}() -function Actor( - thunk; - owner=self(), - children=Dict{String,Mailbox}(), - name=string(nameof(thunk)), - path=(isnothing(_self()) ? "/user" : _self().name) * "/" * name, - mailbox=nothing, - mailbox_size=DEFAULT_MAILBOX_SIZE, -) - return Actor( - path, - thunk, - owner, - children, - Ref{Task}(), - isnothing(mailbox) ? Ref{Mailbox}() : Ref{Mailbox}(mailbox), - mailbox_size - ) -end - -function act(A) - logger = isassigned(LOGGER) ? RemoteLogger(LOGGER[], Logging.Debug) : global_logger() - with_logger(logger) do - handler = A.thunk() - while true - try - msg = take!(A.mailbox[]) - handle(handler, msg) - msg isa StopMsg && break - catch exec - @error exec - for (exc, bt) in Base.catch_stack() - showerror(stdout, exc, bt) - println(stdout) - end - action = A.owner(FailureMsg(exec))[] - if action isa ResumeMsg - handle(handler, action) - continue - elseif action isa StopMsg - handle(handler, action) - rethrow() - elseif action isa RestartMsg - handle(handler, PreRestartMsg()) - handler = A.thunk() - handle(handler, PostRestartMsg()) - else - @error "unknown msg received from $(dirname(nameof(A))): $exec" - rethrow() - end - end +function register(p::Pot) + POT_REGISTRY_CACHE[p.pid] = p + if myid() != 1 + remotecall_wait(1) do + Oolong.POT_REGISTRY_CACHE[p.pid] = p end end + p end -""" -Get the [`Mailbox`](@ref) in the current task. - -!!! note - `self()` in the REPL is bind to `USER`. -""" -function self() - A = _self() - return isnothing(A) ? USER[] : A.mailbox[] -end - -function _self() - try - task_local_storage(ACTOR_KEY) - catch ex - if ex isa KeyError - nothing - else - rethrow() +function link(p::PotID, ch::RemoteChannel) + POT_LINK_CACHE[p] = ch + if myid() != 1 + remotecall_wait(1) do + Oolong.POT_LINK_CACHE[p] = ch end end end -function _schedule(A::Actor) - if !isassigned(A.mailbox) - A.mailbox[] = Mailbox(;size=A.mailbox_size) - end - A.taskref[] = Threads.@spawn begin - task_local_storage(ACTOR_KEY, A) - act(A) - end - return A.mailbox[] -end - -struct ScheduleMsg <: AbstractSysMsg - actor::Actor -end - -function Base.schedule(A::Actor) - s = _self() - if isnothing(s) - if A.owner === NOBODY - _schedule(A) - else - # the actor is submitted from REPL - # we schedule the actor through USER so that it will be bind to USER - USER[](ScheduleMsg(A))[] +function Base.getindex(p::PotID) + get!(POT_LINK_CACHE, p) do + ch = remotecall_wait(1) do + get(Oolong.POT_LINK_CACHE, p, nothing) end - else - if A.owner === ROOT[] - mailbox = _schedule(A) + if isnothing(ch[]) + boil(p[!]) else - mailbox = SCHEDULER[](ScheduleMsg(A))[] + ch[] end - s.children[nameof(A)] = mailbox - mailbox - end -end - - -macro actor(exs...) - a = exs[1] - name = if a isa Symbol - string(a) - elseif a isa Expr && a.head == :call - string(a.args[1]) - else - nothing end - - default_kw = isnothing(name) ? (;) : (;name=name) - thunk = esc(:(() -> ($(a)))) - kwargs = [esc(x) for x in exs[2:end]] - kw = :(merge($default_kw, (;$(kwargs...)))) - - quote - schedule(Actor($thunk; $kw...)) - end -end - -##### -# System Behaviors -##### -function handle(x, args...;kwargs...) - x(args...;kwargs...) -end - -function handle(x, ::FailureMsg) - RestartMsg() end -function handle(x, ::PreRestartMsg) - @debug "stopping children before restart" - handle(x, StopMsg("stop before restart")) +struct PotNotRegisteredError <: Exception + pid::PotID end -function handle(x, ::PostRestartMsg) - @debug "starting after restart signal" - handle(x, StartMsg(:restart)) -end - -function handle(x, msg::StartMsg) - @debug "start msg received" -end +Base.showerror(io::IO, err::PotNotRegisteredError) = print(io, "can not find any pot associated with the pid: $(err.pid)") -function handle(x, msg::StopMsg) - for c in values(_self().children) - c(msg)[] # ??? blocking +function Base.getindex(p::PotID, ::typeof(!)) + get!(POT_REGISTRY_CACHE, p) do + pot = remotecall_wait(1) do + get(Oolong.POT_REGISTRY_CACHE, p, nothing) + end + if isnothing(pot[]) + throw(PotNotRegisteredError(p)) + else + pot[] + end end end -struct ActorStat - path::String -end - -function handle(x, ::StatMsg) - s = _self() - ActorStat( - s.path - ) -end - -Base.stat(m::Mailbox) = m(StatMsg())[] -Base.pathof(m::Mailbox) = stat(m).path -Base.nameof(m::Mailbox) = basename(pathof(m)) - -function handle(::RootActor, s::StartMsg) - @info "$(@__MODULE__) starting..." - LOGGER[] = @actor LOGGER_ACTOR path="/logger" - LOGGER[](s)[] # blocking to ensure LOGGER has started - SCHEDULER[] = @actor SCHEDULER_ACTOR path="/scheduler" - SCHEDULER[](s)[] # blocking to ensure SCHEDULER has started - USER[] = @actor USER_ACTOR path = "/user" - USER[](s)[] # blocking to ensure USER has started -end - -function handle(::LoggerActor, ::StartMsg) - @info "LOGGER started" -end - -function handle(L::LoggerActor, msg::LogMsg) - buf = IOBuffer() - iob = IOContext(buf, stderr) - - level, message, _module, group, id, file, line = msg.args - - color, prefix, suffix = Logging.default_metafmt( - level, _module, group, id, file, line - ) - printstyled(iob, prefix; bold=true, color=color) - printstyled(iob, "[$(msg.kwargs.datetime)]"; color=:light_black) - printstyled(iob, "(@$(msg.kwargs.path))"; color=:green) - print(iob, message) - for (k,v) in pairs(msg.kwargs) - if k ∉ (:datetime, :path) - print(iob, " ") - printstyled(iob, k; color=:yellow) - print(iob, "=") - print(iob, v) +function local_boil(p::Pot) + pid, tea_bag = p.pid, p.tea_bag + ch = RemoteChannel() do + Channel(typemax(Int)) do ch + task_local_storage(KEY, pid) + tea = tea_bag() + while true + flavor = take!(ch) + process(tea, flavor) + end end end - !isempty(suffix) && printstyled(iob, "($suffix)"; color=:light_black) - println(iob) - write(stderr, take!(buf)) + link(pid, ch) + ch end -function handle(::SchedulerActor, ::StartMsg) - @info "SCHEDULER started" -end +boil(p::PotID) = boil(p[!]) +boil(p::Pot) = SCHEDULER(p)[] -function handle(::SchedulerActor, msg::ScheduleMsg) - # TODO: schedule it smartly based on workers' status - @debug "scheduling $(nameof(msg.actor))" - _schedule(msg.actor) +struct Scheduler end -function handle(::UserActor, ::StartMsg) - @info "USER started" -end +(s::Scheduler)(p::Pot) = local_boil(p) -function handle(::UserActor, s::ScheduleMsg) - mailbox = SCHEDULER[](s)[] - _self().children[nameof(s.actor)] = mailbox -end +whereis(p::PotID) = p[].where ##### -# Syntax Sugar -##### - +# Message passing ##### -struct CallMsg{T} - args::T - kwargs - value_box -end - -function handle(x, c::CallMsg) - try - put!(c.value_box, SuccessMsg(handle(x, c.args...; c.kwargs...))) - catch exec - put!(c.value_box, FailureMsg(exec)) - rethrow() - end - nothing +struct CallMsg + args + kw + future end -function (m::Mailbox)(args...;kwargs...) - value_box = FutureWrapper(whereis(m)) - msg = CallMsg(args, kwargs, value_box) - put!(m, msg) - value_box +function (p::PotID)(args...;kw...) + f = Future(whereis(p)) # !!! the result should reside in the same place + put!(p, CallMsg(args, kw.data, f)) + f end -##### - -struct CastMsg{T} - args::T +# ??? non specialized tea? +function process(tea, msg::CallMsg) + res = tea(msg.args...;msg.kw...) + put!(msg.future, res) end -handle(x, c::CastMsg) = handle(x, c.args...) - -function Base.getindex(m::Mailbox, args...) - put!(m, CastMsg(args)) - nothing +function Base.put!(p::PotID, flavor) + try + put!(p[], flavor) + catch e + # TODO add test + if e isa PotNotRegisteredError + rethrow(e) + else + @error e + boil(p) + put!(p, flavor) + end + end end ##### - -struct GetPropMsg - name::Symbol - value_box::FutureWrapper -end - -handle(x, p::GetPropMsg) = put!(p.value_box, getproperty(x, p.name)) - -function Base.getproperty(m::Mailbox, name::Symbol) - res = FutureWrapper(whereis(m)) - put!(m, GetPropMsg(name, res)) - res -end - +# System Initialization ##### -Base.@kwdef struct RequestMsg{M} - msg::M - from::Mailbox = self() -end - -Base.@kwdef struct ReplyMsg{M} - msg::M - from::Mailbox = self() -end - -req(x::Mailbox, msg) = put!(x, RequestMsg(msg=msg)) -rep(x::Mailbox, msg) = put!(x, ReplyMsg(msg=msg)) -async_req(x::Mailbox, msg) = Threads.@spawn put!(x, RequestMsg(msg=msg)) -async_rep(x::Mailbox, msg) = Threads.@spawn put!(x, ReplyMsg(msg=msg)) - -handle(x, req::RequestMsg) = rep(req.from, handle(x, req.msg)) - -# !!! force system level messages to be executed immediately -# directly copied from -# https://github.com/JuliaLang/julia/blob/6aaedecc447e3d8226d5027fb13d0c3cbfbfea2a/base/channels.jl#L13-L31 -# with minor modification -function Base.put_buffered( - c::Channel, - v::Union{ - AbstractSysMsg, - CallMsg{<:Tuple{<:AbstractSysMsg}}, - CastMsg{<:Tuple{<:AbstractSysMsg}} - } -) - lock(c) - try - while length(c.data) == c.sz_max - Base.check_channel_state(c) - wait(c.cond_put) - end - pushfirst!(c.data, v) # !!! force sys msg to be handled immediately - # notify all, since some of the waiters may be on a "fetch" call. - notify(c.cond_take, nothing, true, false) - finally - unlock(c) +struct Root + function Root() + Pot(SCHEDULER, ()->Scheduler()) |> register |> local_boil + new() end - return v end -# !!! This should be called ONLY once -function init() - ROOT[] = @actor ROOT_ACTOR owner=NOBODY path="/" - ROOT[](StartMsg(nothing))[] # blocking is required +function start() + Pot(ROOT, ()->Root()) |> register |> local_boil end diff --git a/src/parameter_server.jl b/src/parameter_server.jl deleted file mode 100644 index aff7237..0000000 --- a/src/parameter_server.jl +++ /dev/null @@ -1,21 +0,0 @@ -struct ParameterServer - params -end - -function (ps::ParameterServer)(gs) - for (p, g) in zip(ps.params, gs) - p .-= g - end -end - -(ps::ParameterServer)() = deepcopy(p.params) - -# Example usage -# ```julia -# ps = @actor ParameterServer([zeros(Float32, 3, 4), zeros(Float32, 3)]) -# for c in clients -# params = ps()[] -# gs = calc_gradients(params) -# ps[gs] -# end -# ``` \ No newline at end of file diff --git a/src/scheduler.jl b/src/scheduler.jl deleted file mode 100644 index 72e288b..0000000 --- a/src/scheduler.jl +++ /dev/null @@ -1,15 +0,0 @@ -#= - -# Design Doc - -- Each node *usually* creates ONE processor. -- Each processor has ONE `LocalScheduler` -- Each `LocalScheduler` sends its available resources to the global `Scheduler` on the driver processor periodically. -- `LocalScheduler` tries to initialize actors on the local processor when available resource meets. Otherwise, send the actor initializer to the global `Scheduler`. -- If there's no resource matches, the actor is put into the `StagingArea` and will be reschedulered periodically. - -Auto scaling: - -- When the whole cluster is under presure, the global `Scheduler` may send request to claim for more resources. -- When most nodes are idle, the global `Scheduler` may ask some nodes to exit and reschedule the actors on it. -=# \ No newline at end of file diff --git a/src/serve.jl b/src/serve.jl deleted file mode 100644 index 92b8316..0000000 --- a/src/serve.jl +++ /dev/null @@ -1,114 +0,0 @@ - -# directly taken from https://github.com/FluxML/Flux.jl/blob/27c4c77dc5abd8e791f4ca4e68a65fc7a91ebcfd/src/utils.jl#L544-L566 -batchindex(xs, i) = (reverse(Base.tail(reverse(axes(xs))))..., i) - -function batch(xs) - data = first(xs) isa AbstractArray ? - similar(first(xs), size(first(xs))..., length(xs)) : - Vector{eltype(xs)}(undef, length(xs)) - for (i, x) in enumerate(xs) - data[batchindex(data, i)...] = x - end - return data -end - -struct ProcessBatchMsg end - -mutable struct BatchStrategy - buffer::Vector{Any} - reqs::Vector{Mailbox} - model::Mailbox - batch_wait_timeout_s::Float64 - max_batch_size::Int - timer::Union{Nothing, Timer} - n_ongoing_batches::Int -end - -""" - BatchStrategy(model;kwargs...) - -# Keyword Arguments - -- `model::Mailbox`, an actor which wraps the model. This actor must accepts - [`RequestMsg`](@ref) as input and reply with a [`ReplyMsg`](@ref) - correspondingly. -- `batch_wait_timeout_s=0.0`, time to wait before handling the next batch. -- `max_batch_size=8, the maximum batch size to handle each time. - -Everytime we processed a batch, we create a timer and wait for at most -`batch_wait_timeout_s` to handle the next batch. If we get `max_batch_size` -requests before reaching `batch_wait_timeout_s`, the timer is reset. If -`batch_wait_timeout_s==0`, we process the available requests immediately. - -!!! warning - The `model` must reply in a non-blocking way (by using [`async_rep`](@ref) or). - Otherwise, there may be deadlock (see test cases if you are interested). -""" -function BatchStrategy( - model; - batch_wait_timeout_s=0.0, - max_batch_size=8, -) - mb = self() - if batch_wait_timeout_s == 0. - timer = nothing - else - timer = Timer(batch_wait_timeout_s) do timer - mb[ProcessBatchMsg()] - end - end - BatchStrategy( - Vector(), - Vector{Mailbox}(), - model, - batch_wait_timeout_s, - max_batch_size, - nothing, - 0 - ) -end - -function reset_timer!(s::BatchStrategy) - isnothing(s.timer) || close(s.timer) - mb = self() - s.timer = Timer(s.batch_wait_timeout_s) do t - mb[ProcessBatchMsg()] - end -end - -function handle(s::BatchStrategy, req::RequestMsg) - push!(s.buffer, req.msg) - push!(s.reqs, req.from) - - if length(s.buffer) == 1 - if s.batch_wait_timeout_s == 0 - if s.n_ongoing_batches == 0 - s(ProcessBatchMsg()) - end - else - reset_timer!(s) # set a timer to insert a ProcessBatchMsg to self() - end - elseif length(s.buffer) == s.max_batch_size - s(ProcessBatchMsg()) - end -end - -function (s::BatchStrategy)(::ProcessBatchMsg) - if !isempty(s.buffer) - @info "???" s.buffer - data = length(s.buffer) == 1 ? reshape(s.buffer[1], size(s.buffer[1])..., 1) : batch(s.buffer) - empty!(s.buffer) - s.n_ongoing_batches += 1 - req(s.model, data) - end -end - -function (s::BatchStrategy)(msg::ReplyMsg) - for res in msg.msg - rep(popfirst!(s.reqs), res) - end - s.n_ongoing_batches -= 1 - s(ProcessBatchMsg()) -end - -# X** \ No newline at end of file From 25153dac0409031e2482175a4ba6d7344c6197cb Mon Sep 17 00:00:00 2001 From: Jun Tian Date: Fri, 3 Sep 2021 18:23:12 +0800 Subject: [PATCH 2/4] add logger --- Project.toml | 1 + README.md | 18 +- src/core.jl | 457 +++++++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 402 insertions(+), 74 deletions(-) diff --git a/Project.toml b/Project.toml index b4f2edd..25e30af 100644 --- a/Project.toml +++ b/Project.toml @@ -4,6 +4,7 @@ authors = ["Jun Tian and contributors"] version = "0.1.0" [deps] +CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" diff --git a/README.md b/README.md index 85a4248..0de70a3 100644 --- a/README.md +++ b/README.md @@ -19,26 +19,37 @@ Users can easily extend existing packages to apply them in a cluster. - Simple API - + Only very minimal APIs are provided to ease the usage. ## Roadmap - Stage 1 - [ ] Stabilize API - - [ ] `@pot`, define a container over any callable object. + - [x] `@pot tea [prop=value...]`, define a container over any callable object. + - [x] `(p::PotID)(args...;kw...)`, async call, at most once delievery, a `Promise` is returned. + - [x] `msg |> p::PotID`, async call, at most once delievery, no return. + - [x] `(p::PotID).prop`, async call, at most once delievery, return the `prop` of the inner `tea`. - [ ] `-->`, `<--`, define a streaming pipeline. + - [ ] Features + - [x] Logging. All messages are sent to primary node by default. + - [ ] RemoteREPL + - [x] CPU/GPU allocation + - [ ] Auto intall+using dependencies - [ ] Example usages - Stage 2 - - [ ] Auto-scaling. Allow workers join/exit. + - [ ] Auto-scaling. Allow workers join/exit? - [ ] Custom cluster manager - [ ] Dashboard - [ ] [grafana](https://grafana.com/) - [ ] Custom Logger - [LokiLogger.jl](https://github.com/fredrikekre/LokiLogger.jl) - [Stipple.jl](https://github.com/GenieFramework/Stipple.jl) + - [ ] Tracing - Stage 3 - [ ] Drop out Distributed.jl? - [ ] K8S + - [ ] differentiate accross pots? + - [ ] Python client (transpile, pickle) ## Design @@ -52,6 +63,7 @@ | PotID |<===>| | +---+---+ | PotID | | | () -> Tea | + | | require | | +-------------+ +-------|-------------------------+ | V boiled somewhere | diff --git a/src/core.jl b/src/core.jl index 067f5a7..58d7ddd 100644 --- a/src/core.jl +++ b/src/core.jl @@ -2,40 +2,17 @@ export @P_str, @pot using Distributed using UUIDs:uuid4 +using Base.Threads +using CUDA +using Logging +using Dates const KEY = :OOLONG -##### -# Pot Definition -##### - struct PotID path::Tuple{Vararg{Symbol}} end -struct Pot - pid::PotID - tea_bag::Any -end - -function Pot( - tea_bag; - name=string(uuid4()) -) - pid = PotID(name) - Pot(pid, tea_bag) -end - -macro pot(tea, kw...) - tea_bag = esc(:(() -> ($(tea)))) - xs = [esc(x) for x in kw] - quote - p = Pot($tea_bag; $(xs...)) - register(p) - p.pid - end -end - """ P"[/]your/pot/path" @@ -49,10 +26,10 @@ macro P_str(s) end function Base.show(io::IO, p::PotID) - if isempty(p.path) + if isempty(getfield(p, :path)) print(io, '/') else - for x in p.path + for x in getfield(p, :path) print(io, '/') print(io, x) end @@ -64,30 +41,192 @@ function PotID(s::String) if s[1] == '/' PotID(Tuple(Symbol(x) for x in split(s, '/';keepempty=false))) else - PotID((self().path..., (Symbol(x) for x in split(s, '/';keepempty=false))...)) + self() * s end else PotID(()) end end +Base.:(*)(p::PotID, s::String) = PotID((getfield(p, :path)..., (Symbol(x) for x in split(s, '/';keepempty=false))...)) + const ROOT = P"/" -const USER = P"/user" +const LOGGER = P"/log" const SCHEDULER = P"/scheduler" +const USER = P"/user" + + +# struct Success{V} +# value::V +# end + +# Base.getindex(s::Success{<:Success}) = s.value[] +# Base.getindex(s::Success) = s.value -self() = try - task_local_storage(KEY) -catch - USER +# struct Failure{E} +# error::E +# end + +# Failure() = Failure(nothing) + +# Base.getindex(f::Failure{<:Failure}) = f.error[] +# Base.getindex(f::Failure) = f.error + +"Similar to `Future`, but it will unwrap inner `Future` or `Promise` recursively." +struct Promise + f::Future + function Promise(args...) + new(Future(args...)) + end end +function Base.getindex(p::Promise) + x = p.f[] + while x isa Promise || x isa Future + x = x[] + end + x +end + +Base.put!(p::Promise, x) = put!(p.f, x) + +##### +# Logging +##### + +Base.@kwdef struct DefaultLogger <: AbstractLogger + min_level::LogLevel = Logging.Debug + date_format::Dates.DateFormat=Dates.default_format(DateTime) +end + +Logging.shouldlog(::DefaultLogger, args...) = true +Logging.min_enabled_level(L::DefaultLogger) = L.min_level + +const DEFAULT_LOGGER = DefaultLogger() + +struct LogMsg + args + kw +end + +function (L::DefaultLogger)(msg::LogMsg) + args, kw = msg.args, msg.kw + + buf = IOBuffer() + iob = IOContext(buf, stderr) + + level, message, _module, group, id, file, line = args + + color, prefix, suffix = Logging.default_metafmt( + level, _module, group, id, file, line + ) + + printstyled(iob, prefix; bold=true, color=color) + printstyled(iob, "$(kw.datetime)"; color=:light_black) + printstyled(iob, "[$(kw.from)@$(kw.myid)]"; color=:green) + print(iob, message) + for (k,v) in pairs(kw) + if k ∉ (:datetime, :path, :myid, :from) + print(iob, " ") + printstyled(iob, k; color=:yellow) + printstyled(iob, "="; color=:light_black) + print(iob, v) + end + end + !isempty(suffix) && printstyled(iob, "($suffix)"; color=:light_black) + println(iob) + write(stderr, take!(buf)) +end + +function Logging.handle_message(logger::DefaultLogger, args...; kw...) + kw = merge( + kw.data, + ( + datetime="$(Dates.format(now(), logger.date_format))", + from=self(), + myid=myid(), + ) + ) + LogMsg(args, kw) |> LOGGER +end + +##### +# Pot Definition +##### + +struct RequireInfo + cpu::Float64 + gpu::Float64 +end + +Base.:(<=)(x::RequireInfo, y::RequireInfo) = x.cpu <= y.cpu && x.gpu <= y.gpu +Base.:(-)(x::RequireInfo, y::RequireInfo) = RequireInfo(x.cpu - y.cpu, x.gpu - y.gpu) + +struct Pot + tea_bag::Any + pid::PotID + require::RequireInfo + logger::Any +end + +function Pot( + tea_bag; + name=string(uuid4()), + cpu=eps(), + gpu=0, + logger=DEFAULT_LOGGER +) + pid = name isa PotID ? name : PotID(name) + require = RequireInfo(cpu, gpu) + Pot(tea_bag, pid, require, logger) +end + +macro pot(tea, kw...) + tea_bag = esc(:(() -> ($(tea)))) + xs = [esc(x) for x in kw] + quote + p = Pot($tea_bag; $(xs...)) + register(p) + p.pid + end +end + +struct PotState + pid::PotID + task::Task +end + +_self() = get!(task_local_storage(), KEY, PotState(USER, current_task())) +self() = _self().pid + +local_scheduler() = SCHEDULER*"local_scheduler_$(myid())" + Base.parent() = parent(self()) -Base.parent(p::PotID) = PotID(p.path[1:end-1]) +Base.parent(p::PotID) = PotID(getfield(p, :path[1:end-1])) + +##### +# Exceptions +##### + +struct PotNotRegisteredError <: Exception + pid::PotID +end + +Base.showerror(io::IO, err::PotNotRegisteredError) = print(io, "can not find any pot associated with the pid: $(err.pid)") + +struct RequirementNotSatisfiedError <: Exception + required::RequireInfo + remaining::RequireInfo +end + +Base.showerror(io::IO, err::RequirementNotSatisfiedError) = print(io, "required: $(err.required), remaining: $(err.remaining)") ##### # Pot Scheduling ##### +# TODO: set ttl? + "local cache to reduce remote call, we may use redis like db later" const POT_LINK_CACHE = Dict{PotID, RemoteChannel{Channel{Any}}}() const POT_REGISTRY_CACHE = Dict{PotID, Pot}() @@ -117,18 +256,14 @@ function Base.getindex(p::PotID) get(Oolong.POT_LINK_CACHE, p, nothing) end if isnothing(ch[]) - boil(p[!]) + boil(p) else ch[] end end end -struct PotNotRegisteredError <: Exception - pid::PotID -end - -Base.showerror(io::IO, err::PotNotRegisteredError) = print(io, "can not find any pot associated with the pid: $(err.pid)") +whereis(p::PotID) = p[].where function Base.getindex(p::PotID, ::typeof(!)) get!(POT_REGISTRY_CACHE, p) do @@ -143,15 +278,26 @@ function Base.getindex(p::PotID, ::typeof(!)) end end +Base.getindex(p::PotID, ::typeof(*)) = p(_self())[] + +local_boil(p::PotID) = local_boil(p[!]) + function local_boil(p::Pot) - pid, tea_bag = p.pid, p.tea_bag + pid, tea_bag, logger = p.pid, p.tea_bag, p.logger ch = RemoteChannel() do - Channel(typemax(Int)) do ch - task_local_storage(KEY, pid) - tea = tea_bag() - while true - flavor = take!(ch) - process(tea, flavor) + Channel(typemax(Int),spawn=true) do ch + task_local_storage(KEY, PotState(pid, current_task())) + with_logger(logger) do + tea = tea_bag() + while true + try + flavor = take!(ch) + process(tea, flavor) + catch err + @error err + finally + end + end end end end @@ -159,38 +305,154 @@ function local_boil(p::Pot) ch end -boil(p::PotID) = boil(p[!]) -boil(p::Pot) = SCHEDULER(p)[] +"blocking until a valid channel is established" +boil(p::PotID) = local_scheduler()(p)[] + +struct CPUInfo + total_threads::Int + allocated_threads::Int + total_memory::Int + free_memory::Int + function CPUInfo() + new( + Sys.CPU_THREADS, + Threads.nthreads(), + convert(Int, Sys.total_memory()), + convert(Int, Sys.free_memory()), + ) + end +end + +struct GPUInfo + name::String + total_memory::Int + free_memory::Int + function GPUInfo() + new( + name(device()), + CUDA.total_memory(), + CUDA.available_memory() + ) + end +end -struct Scheduler +struct ResourceInfo + cpu::CPUInfo + gpu::Vector{GPUInfo} end -(s::Scheduler)(p::Pot) = local_boil(p) +function ResourceInfo() + cpu = CPUInfo() + gpu = [] + if CUDA.functional() + for d in devices() + device!(d) do + push!(gpu, GPUInfo()) + end + end + end + ResourceInfo(cpu, gpu) +end -whereis(p::PotID) = p[].where +Base.convert(::Type{RequireInfo}, r::ResourceInfo) = RequireInfo(r.cpu.allocated_threads, length(r.gpu)) -##### -# Message passing -##### +struct HeartBeat + resource::ResourceInfo + available::RequireInfo + from::PotID +end -struct CallMsg - args - kw - future +struct LocalScheduler + pending::Dict{PotID, Future} + peers::Ref{Dict{PotID, RequireInfo}} + available::Ref{RequireInfo} + timer::Timer end -function (p::PotID)(args...;kw...) - f = Future(whereis(p)) # !!! the result should reside in the same place - put!(p, CallMsg(args, kw.data, f)) - f +# TODO: watch exit info + +function LocalScheduler() + pid = self() + req = convert(RequireInfo, ResourceInfo()) + available = Ref(req) + timer = Timer(1;interval=1) do t + HeartBeat(ResourceInfo(), available[], pid) |> SCHEDULER # !!! non blocking + end + + pending = Dict{PotID, Future}() + peers = Ref(Dict{PotID, RequireInfo}(pid => req)) + + LocalScheduler(pending, peers, available, timer) end -# ??? non specialized tea? -function process(tea, msg::CallMsg) - res = tea(msg.args...;msg.kw...) - put!(msg.future, res) +function (s::LocalScheduler)(p::PotID) + pot = p[!] + if pot.require <= s.available[] + res = local_boil(p) + s.available[] -= pot.require + res + else + res = Future() + s.pending[p] = res + res + end +end + +function (s::LocalScheduler)(peers::Dict{PotID, RequireInfo}) + s.peers[] = peers + for (p, f) in s.pending + pot = p[!] + for (w, r) in peers + if pot.require <= r + # transfer to w + put!(f, w(p)) + delete!(s.pending, p) + break + end + end + end +end + +Base.@kwdef struct Scheduler + workers::Dict{PotID, HeartBeat} = Dict() + pending::Dict{PotID, Future} = Dict() +end + +# ??? throttle +function (s::Scheduler)(h::HeartBeat) + # ??? TTL + s.workers[h.from] = h + + for (p, f) in s.pending + pot = p[!] + if pot.require <= h.available + put!(f, h.from(p)) + end + end + + Dict( + p => h.available + for (p, h) in s.workers + ) |> h.from # !!! non blocking end +# pots are all scheduled on workers only +function (s::Scheduler)(p::PotID) + pot = p[!] + for (w, h) in s.workers + if pot.require <= h.available + return w(p) + end + end + res = Future() + s.pending[p] = res + res +end + +##### +# Message passing +##### + function Base.put!(p::PotID, flavor) try put!(p[], flavor) @@ -206,17 +468,70 @@ function Base.put!(p::PotID, flavor) end end +process(tea, flavor) = tea(flavor) + +# CallMsg + +struct CallMsg{A} + args::A + kw + promise +end + +function (p::PotID)(args...;kw...) + promise = Promise(whereis(p)) # !!! the result should reside in the same place + put!(p, CallMsg(args, kw.data, promise)) + promise +end + +# ??? non specialized tea? +function process(tea, msg::CallMsg) + try + res = handle(tea, msg.args...;msg.kw...) + put!(msg.promise, res) + catch err + # avoid dead lock + put!(msg.promise, err) + end +end + +handle(tea, args...;kw...) = tea(args...;kw...) + +# CastMsg + +Base.:(|>)(x, p::PotID) = put!(p, x) + +# GetPropMsg + +struct GetPropMsg + prop::Symbol +end + +Base.getproperty(p::PotID, prop::Symbol) = p(GetPropMsg(prop)) + +handle(tea, msg::GetPropMsg) = getproperty(tea, msg.prop) + ##### # System Initialization ##### +## Root + struct Root function Root() - Pot(SCHEDULER, ()->Scheduler()) |> register |> local_boil + local_boil(@pot DefaultLogger() name=LOGGER logger=current_logger()) + local_boil(@pot Scheduler() name=SCHEDULER) new() end end function start() - Pot(ROOT, ()->Root()) |> register |> local_boil + @info "$(@__MODULE__) starting..." + if myid() == 1 + local_boil(@pot Root() name=ROOT logger=current_logger()) + end + + if myid() in workers() + local_boil(@pot LocalScheduler() name=local_scheduler()) + end end From 289d7701c048a5ef8d7eac1f1081d88a197fd2fd Mon Sep 17 00:00:00 2001 From: Jun Tian Date: Fri, 10 Sep 2021 17:32:58 +0800 Subject: [PATCH 3/4] sync --- Project.toml | 3 +- README.md | 151 ++++++++++------ docs/logo.jl | 42 +++++ docs/logo.png | Bin 0 -> 40636 bytes docs/logo.svg | 13 ++ src/Oolong.jl | 5 +- src/config.jl | 7 + src/core.jl | 440 +++++++++++++++++++++++++++++++++-------------- test/runtests.jl | 1 - test/serve.jl | 52 ------ 10 files changed, 468 insertions(+), 246 deletions(-) create mode 100644 docs/logo.jl create mode 100644 docs/logo.png create mode 100644 docs/logo.svg create mode 100644 src/config.jl delete mode 100644 test/serve.jl diff --git a/Project.toml b/Project.toml index 25e30af..29fd500 100644 --- a/Project.toml +++ b/Project.toml @@ -5,11 +5,12 @@ version = "0.1.0" [deps] CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" +Configurations = "5218b696-f38b-4ac9-8b61-a12ec717816d" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" -URIs = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6" [compat] julia = "1" diff --git a/README.md b/README.md index 0de70a3..07bb244 100644 --- a/README.md +++ b/README.md @@ -1,58 +1,102 @@ -# Oolong.jl +
+Oolong.jl logo
+  ____        _                     |  > 是非成败转头空
+ / __ \      | |                    |  > Success or failure,
+| |  | | ___ | | ___  _ __   __ _   |  > right or wrong,
+| |  | |/ _ \| |/ _ \| '_ \ / _` |  |  > all turn out vain.
+| |__| | (_) | | (_) | | | | (_) |  |
+ \____/ \___/|_|\___/|_| |_|\__, |  |  The Immortals by the River 
+                             __/ |  |  -- Yang Shen 
+                            |___/   |  (Translated by Xu Yuanchong) 
+
+ +**Oolong.jl** is a framework for building scalable distributed applications in Julia. -*An actor framework for [~~ReinforcementLearning.jl~~](https://github.com/JuliaReinforcementLearning/ReinforcementLearning.jl) distributed computing in Julia.* +## Features -> “是非成败转头空” —— [《临江仙》](https://www.vincentpoon.com/the-immortals-by-the-river-----------------.html) -> [杨慎](https://zh.wikipedia.org/zh-hans/%E6%9D%A8%E6%85%8E) -> -> "Success or failure, right or wrong, all turn out vain." - [*The Immortals by -> the -> River*](https://www.vincentpoon.com/the-immortals-by-the-river-----------------.html), -> [Yang Shen](https://en.wikipedia.org/wiki/Yang_Shen) -> -> (Translated by [Xu Yuanchong](https://en.wikipedia.org/wiki/Xu_Yuanchong)) +- Easy to use + Only very minimal APIs are exposed to make this package easy to use (yes, easier than [Distributed.jl](https://docs.julialang.org/en/v1/stdlib/Distributed/)). +- Non-invasive + Users can easily extend existing packages to apply them in a cluster. -## Features +- Fault tolerance -- Non-invasive - Users can easily extend existing packages to apply them in a cluster. +- Auto scaling + +## Get Started + +⚠ *This package is still under rapid development and is not registered yet.* + +First install this package: + +```julia +pkg> activate --temp + +pkg> add https://github.com/JuliaReinforcementLearning/Oolong.jl +``` + +See tests for some example usages. (TODO: move typical examples here when APIs are stabled) -- Simple API - Only very minimal APIs are provided to ease the usage. +## Examples + +- Batch evaluation. +- AlphaZero +- Parameter server +- Parameter search + +Please contact us if you have a concrete scenario but not sure how to use this package! + +## Deployment + +### Local Machines + +### K8S ## Roadmap -- Stage 1 - - [ ] Stabilize API - - [x] `@pot tea [prop=value...]`, define a container over any callable object. - - [x] `(p::PotID)(args...;kw...)`, async call, at most once delievery, a `Promise` is returned. - - [x] `msg |> p::PotID`, async call, at most once delievery, no return. - - [x] `(p::PotID).prop`, async call, at most once delievery, return the `prop` of the inner `tea`. - - [ ] `-->`, `<--`, define a streaming pipeline. - - [ ] Features - - [x] Logging. All messages are sent to primary node by default. - - [ ] RemoteREPL - - [x] CPU/GPU allocation - - [ ] Auto intall+using dependencies - - [ ] Example usages -- Stage 2 - - [ ] Auto-scaling. Allow workers join/exit? - - [ ] Custom cluster manager - - [ ] Dashboard - - [ ] [grafana](https://grafana.com/) - - [ ] Custom Logger - - [LokiLogger.jl](https://github.com/fredrikekre/LokiLogger.jl) - - [Stipple.jl](https://github.com/GenieFramework/Stipple.jl) - - [ ] Tracing -- Stage 3 - - [ ] Drop out Distributed.jl? - - [ ] K8S - - [ ] differentiate accross pots? - - [ ] Python client (transpile, pickle) +1. Stage 1 + 1. Stabilize API + 1. ☑️ `p::PotID = @pot tea [prop=value...]`, define a container over any callable object. + 2. ☑️ `(p::PotID)(args...;kw...)`, which behaves just like `tea(args...;kw...)`, except that it's an async call, at most once delievery, a `Promise` is returned. + 3. ☑️ `msg |> p::PotID` similar to the above one, except that nothing is returned. + 4. ☑️ `(p::PotID).prop`, async call, at most once delievery, return the `prop` of the inner `tea`. + 5. 🧐 `-->`, `<--`, define a streaming pipeline. + 6. 🧐 timed wait on `Promise`. + 2. Features + 1. ☑️ Logging. All messages are sent to primary node by default. + 2. 🧐 RemoteREPL + 3. ☑️ CPU/GPU allocation + 4. 🧐 Auto intall+using dependencies + 5. ☑️ Global configuration + 3. Example usages + 1. 🧐 Parameter search + 2. 🧐 Batch evaluation. + 3. 🧐 AlphaZero + 4. 🧐 Parameter server +2. Stage 2 + 1. Auto1.scaling. Allow workers join/exit? + 1. 🧐 Custom cluster manager + 2. Dashboard + 1. 🧐 [grafana](https://grafana.com/) + 3. Custom Logger + 1. 🧐 [LokiLogger.jl](https://github.com/fredrikekre/LokiLogger.jl) + 2. 🧐 [Stipple.jl](https://github.com/GenieFramework/Stipple.jl) + 4. Tracing +1. Stage 3 + 1. Drop out Distributed.jl? + 1. 🧐 `Future` will transfer the ownership of the underlying data to the caller. Not very efficient when the data is passed back and forth several times in its life circle. + 2. 🧐 differentiate across pots? + 3. 🧐 Python client (transpile, pickle) + 4. 🧐 K8S + 5. 🧐 JuliaHub + 6. 🧐 AWS + 7. 🧐 Azure ## Design +### Workflow + ``` +--------+ | Flavor | @@ -78,33 +122,26 @@ | | +-----------+ | | V | | +----+----+ | - | | Future | | + | | Promise | | | +---------+ | +---------------------------------+ ``` -A `Pot` is mainly a container of an arbitrary object (`tea`) which is instantiated by calling a parameterless function. Whenever a `Pot` receives a `flavor` through the `channel`, the water in the `Pot` is *boiled* first (a `task` to process `tea` and `flavor` is created) if it is cool (the previous `task` was exited by accident or on demand). Users can define how `tea` and `flavor` are processed through multiple dispatch on `process(tea, flavor)`. In some `task`s, users may create many other `Pot`s whose references (`PotID`) are stored in `Children`. A `PotID` is simply a path used to locate a `Pot`. - -## Get Started - -⚠ *This package is still under rapid development and is not registered yet.* - -First install this package: +A `Pot` is mainly a container of an arbitrary object (`tea`) which is instantiated by calling a parameterless function. Whenever a `Pot` receives a `flavor`, the water in the `Pot` is *boiled* first (a `task` is created to process `tea` and `flavor`) if it is cool (the previous `task` was exited by accident or on demand). Some `Pot`s may have a few specific `require`ments (the number of cpu, gpu). If those requirements can not be satisfied, the `Pot` will be pending to wait for new resources. Users can define how `tea` and `flavor` are processed through multiple dispatch on `process(tea, flavor)`. In some `task`s, users may create many other `Pot`s whose references (`PotID`) are stored in `Children`. A `PotID` is simply a path used to locate a `Pot`. -```julia -pkg> activate --temp +### Decisions -pkg> add https://github.com/JuliaReinforcementLearning/Oolong.jl -``` +The following design decisions need to be reviewed continuously. +1. Each `Pot` can only be created inside of another `Pot`, which forms a child-parent relation. If no `Pot` is found in the `current_task()`, the parent is bind to `/user` by default. ### FAQ ## Acknowledgement -This package is mainly inspired by the following packages: +This package is mainly inspired by the following projects: +- [Orleans](https://github.com/dotnet/orleans) - [Proto.Actor](https://proto.actor/) - [Ray](https://ray.io/) -- [Orleans](https://github.com/dotnet//orleans) - [Actors.jl](https://github.com/JuliaActors/Actors.jl) diff --git a/docs/logo.jl b/docs/logo.jl new file mode 100644 index 0000000..5384962 --- /dev/null +++ b/docs/logo.jl @@ -0,0 +1,42 @@ +using Luxor + +scale = 2 +ratio = (√5 -1)/2 + +w, h = scale * 128 * 2, scale * 128 * 2 + +r1 = w/2 +r2 = r1 * ratio +r3 = r2 * ratio +r4 = r3 * ratio + +c1 = Point(0, 0) +c2 = c1 + Point(r1-r2, r1-r2) * √2/2 +c3 = c1 + Point(r1-r3, r1-r3) * √2/2 +c4 = c1 + Point(r1-r4, r1-r4) * √2/2 + +Drawing(w, h, "logo.svg") +background(1, 1, 1, 0) +Luxor.origin() + +setcolor(1,1,1) +circle(c1, r1, :fill) +setcolor(0.251, 0.388, 0.847) # dark blue +circle(c1, r1-4*scale, :fill) + +setcolor(1,1,1) +circle(c2, r2, :fill) +setcolor(0.796, 0.235, 0.2) # dark red +circle(c2, r2-4*scale, :fill) + +setcolor(1,1,1) +circle(c3, r3, :fill) +setcolor(0.22, 0.596, 0.149) # dark green +circle(c3, r3-4*scale, :fill) + +setcolor(1,1,1) +circle(c4, r4, :fill) +setcolor(0.584, 0.345, 0.698) # dark purple +circle(c4, r4-4*scale, :fill) + +finish() \ No newline at end of file diff --git a/docs/logo.png b/docs/logo.png new file mode 100644 index 0000000000000000000000000000000000000000..6dd647055a768d86bd515187459149a295177965 GIT binary patch literal 40636 zcmXt91yojDvwrC=36U-Zq@_DVx>1nsP66qZl#uRJN~F8tmG15?>F(xkzJJ{%Ygy-< znLR!8%>J&VAc=uWgbIN`Fr=l#RUi;p@JCn(G9vix+;!p>{Dx>KD=7|ndj6HwoF5B; zP(Y-`ML)Wx?Jv3{ef)O;JvriMpZ73J%ta3QhWu77yqSBy_46;o?_Ih|*>Bhe&G$J+ zXJoPY3Gf=bU_Xn%Ulb zrMbtEg-&;L_jdp8?d{)EG+z>m#v>2+8)-aZFDopPaGq4NfQ6ZGSq^xXcC(0t>x*i9vihlMpGuQc5LmOI-gzqm* zb3fW6JK|CmNW)Nw2bdQZ`hJHx3tzXu{bY8-5n}l%wxXqfgVj>{Q3fIC9UEngA09+3 zpukSfTY&WfNss>pkgRD|CAF49qGX63KR=<%A zJ{;lu0CyxIBm&q>n0^yv(U3<#u>;Gz_{>ia;sNUc;fHg3pAtv-TVVxF4B>A+lvbvo zCH$8Q#1mIe-OR2@VjAobr$+Au zw50>gtc%E_QoIe*S)X%{`dO}h=MT<#5$7BqB;2>BHIlg;D)%=`=X(YZ?=npCDTWmc zGSb&Pk?Ln_%%7sg#jroaHjQ6hd{_D7Vy^mv%6aY4UoqtGn5H5OCgsG+6KoI3T=--E z-Z)3?ow@DR_{47hPNQD!jkzPD`O*3K5NvA~|KChHx$E0Kw@pY0C+i`-5oYH za9Tvo6I^ZuR;SAI8@V;Tk~)=%o6b*90ulMBxmiU$e~(X{f)ewUKM~vtH3_8ehC00) z3%*~sz&$2led#LcjwD|3>NJy;l1kJ{2bKgwj+P^X{H*$ltkhJsIXL30F4vUX%*0i5 zOJ`z5#JvbTLVT`8I9bNu?Y{G)*2q776oNKVLN)HbwSMY;>hUTz@1p~-qiKK=KWpx( ztk|!Ji|O36r8aFjAZe(t8(icq9TDGuBKA0W{aM}XSNS~NWc%36pSS+XpLW8@<9nTr zD(QNvQ}sxAVZ<2FGLSM?1!2*s@$vws&o{icgc012ynZM-#LpTI%~-ZEx`sP5DH2DCN%%inTm>o zq5=u2o91?Wl3H;xJkD8O$E>i9NPS(X<5@X1m@?Sx+H`81DfMm)e_hh#orN}mYL{8% zLmgv96J1ID`@)_Dp&2B+6W=FHMXv>bXXIJpyJ z=FQ)+S7TsB(Cb?kuN9~!wvTWkuFTC(%JnU{Gnw`bdM&w|$=o_#KE=ly81->!?6-I^ zH!2SO{6%A#Etd%!2{ZgQw;}uDjo4@4fQ0Wf_iX%LLF+}7X^u!I%A6fp&Jvv}^`)__ zxUTwC1PnAxG>AOphD_$-w4GICD8+~V?!tpAyR9=phMNzOK9dT29=_BqBjwr>` zk%XCgm9BwdkpwO$JL-!(jV3dn&~<@R3A*%{l%}61@lg^>ZUmzez+k~D(&sKjx71@l zM1yYxOf{g^`6{*R5ft3Hd!?`>UL9w$G+v+Y*}5-`RkQq}<(}^v!gZW6a$xq5AAHR~ z{fhxD1sNaqDXtkooKXsf;+Kq7Gp;?^&)1$fv;M+_0i|jUt{#4|iY>cclME3pA7(j*DPFW)2a;V||BcST{vBscD31Jy5}SDbO~7M~aP zOl3AShqE)g>%}+eO}xv}UX?5_YVoN4{OKFxaT6e}=Ps|0T%z>VPm*((f zkV6%jui|S#R9^w}h;mjsa=%lb472phuI82YRwuFeUCj*ga81TS9z>9tL{oF8^Vb46D55C2_R3EecL){}oNa}AW8+6~L$_Wt-r z@<0_T+W0amT4+0R#dd`p%H$wkj+}@GX{%^icVU5LHi}ZbFDCWgY+C-iNA5B*`g-r; z!!G?mrDMlZAW5ShW&Q$n-uAD`&qaPZI{2}qj_STlJ_OvU0 zpf%i5MY5|vdmg;IqnK_MEP7M9uU-0fS2~eUc^Uexl}g-RIaJ7KX^X}KyKgH5-@ksk zg45P?*ve?N!ggo#NYHbo0`nWAE!IrI4ZjjmWWRs>O{6p1Ig&rMf`p|}n-<=6D|kd8?6Vl4PW zNDcY6p!Olfnb6nMg?Sr}>fRthG4zjL{QR>o=~u4dLVNnlu{PBh{21a?&Zy#2q+mhH zgj7x8qI`WtCN$zv#rrVeT^TVg?68)w?i?7Bv@b_2xvvB54JJ^H6aJ0j@95Vbr?U6_ zroIX!ho|_$eg#p?;{wY(KW69;ZJAn@?X2u@Ynn_8QLcSf zZt+*eWH7yL353%T+D3eYU-%gp^H#W58YXI^YKk9%1*eMS@NoZnae=$>`z--)sCtOF zvf9QD#2|qVJR=`|ZW42u2Ueb->X>Vxp$p?N%uDJ?p0V8sP+PG#OZY zMii7buTY|o@elRkh>+$xY0qKd^rImnh3`MBkG7Ozr_qacrt11JVe-O{ai>$}!X z59y5!3TIgSD$(e=36yef?;q#gs#CdKbQ78P;eb_V$ju11*EB?yf{9wWbw2fg@Vr~F znx1y8J4$}jqPY|5X($#_S98`u!Yoh_KKcoc@7p!LSToE8j)H7Q6dnXc&dQ7NXZd4{ zeaYD^hvKl62~gYhg9BG7c0r%5>ish1hk+zrz9tc0Y;PRkI6h_d_irf8RuSo?eIVo1ny>vf~ z0N(c2Zqkj>M6IMVZ+=a(+dypLlSNCvFon0|J)`Z!zN+ymXwP(B06GCvnmF!jtq4+B zTXQ$E4fT7=5gfJsUAkh(Mx6+l%)Gg2UU@-+(j!>+7}td)%yEQ(3`BD;8X5}~TGgK3 zJa-%(m%letR(E=VD@Ic+_-4!D#sWVMKd{*a3Jb*GkSCHB3rjI%QLG4gM8MN(MO@Mt z>Gc}s8l9vLtiv|ccc`#Vb918aCQ}MvYMN?8(f%Emf}cJhP9p__onJL|1NDMh074+x zA4PH6N4mVE@WwX?- zVD+?lQ;J)A)rPHTbjPig1y9D^BVFgJ{9%(RKncr%5E=rV`o8oVRV%eHKl!pweZi=; z{zDV!$DlKSE&2X1O(rNYxLQye!~UDfZY5prYsM&8ePkeRTBoSUf8JGCt@~@2w8Y!a zz0u8LA=gpqChPaO{X7mwZT8G(*qW5bO5jj$03le-J#;Z6gVq{%`BK^*8%2)OrLFW) zSvh1%#MKqS)B3e4)OTa;DHPXu_8BTN4ID-wxDbwOs>86rHGXc+tLg2(cq%gTilBA> zjzTF8A6_uDQu-1z{|_TmTE7t74`mrnaeAf`cAzE7;tQIUulhNYRs_rkt6jl+I}W{b zKnjLELls{RG~(|KyBF-43lV;Nl<^|Xt?Ez&!lTyQgY$b?R@b6u<+?e@Mdn_hp>PB& zjwE*_=Y1G(*b4g^oI|QAF@4?hceu;ZuoQ?@Q)VybhL=?`lln8&tqn&m}`%S&g6`Pc~8riu3Vo}gt1<!Rr6;-6t8-kS7kH-ym=D^nD)p1;a3D^|6qseF zemaeT)%UHMQirMi{QcsFlUsNS>XF$i?G0SUIJ3Y_+r#&Ez8 zd3Da8ufJh^;4Kb#jUXaiHI)eCqWN8?zC5w;sOV>*B#{r8<>aL?68p4PrAV(=6a^sp zs+v@0WdhX>k7#Bh3it&IXM&ER?r=%dU9h6G6e3s&D3qCOJlvZ(#F7tIpQuAAU=F*>#oxY_|Al+x3M0!sMpv+zbedeYlFHF3=X8 z3gq5*%H3z>D`VPjt#eo~gYBYNqHM!MAhsdhhVg8j*Cuk!D;o<1Uo;*$`U_?73ZV+* z95@1M6VgUXr|s_x``Kcx?Kl)4;GsFF4hIWgFOVhFH8e?A;4z)H46try>l_jTGqSe0XypC4pX9r6__{7OY}v6E_~}k)WU{WvaPns82>f z^pwXXtiGg79r1kd=pQ8z^`-JaL@VO5c-T^l;;>(7Y74SEup(CJm;zYVMg;RqJ6K!f zN!TDTp`O9mXswtOSO3!qj8X0*fM{AF2%zuV5$x?_+IQM;T3bXlgOgI}PI9@`@BSbv zJgdA!Bs;P~Q&9mALouKhrlNv}6zai`c6crDsOC<{kh0T``il($zC67asWeR8wqtvve>&YO+7cJ0- z%`f(QxEN#CXq4ixnhkS<5s?$w)fT^D$;LFv2_K5qWM%L{E%6;n0(C0xdC$UmcGxzw zP5T-IwYNg|zrwn8Ey`b{AU6H3UJ~k~?~IS52bF$&K*AkUqx_y>{YY zrTvtElqU`YSwPx1B&hm1)vovWNs_``$eadn!FCOoBD|#6-N2Nf=6Ha)K>9CkO>FEX z;Mviya&S$P|6beC1w+u5cNWMkCZdqLhx2M;WWm-VJL{}!D(^tzm+Y>IuNTt2Cp@9^ zR-L=Qw)>gslx87jAhsCI?_x<4hpS2Io<3Hu#{+~*6&bHr{#8NZW0w%~Vp(4^Hj8_I z(d+s57yro*^>bxGqOh>O#c%G|kv`us!W>STmN#d zC(`A!)5Mg3Z<50eaD-{;t*_S5n$%}vi{<>`e_HHG8B7TBDV@~J#6IveWGX-fI_84n zCdt|EJ0M)Q3tOJJnsfpC&3PNCezt|QyX~A@gMjKI9Zr_Z!GOJt9VztsF!8zcC z7;-c$0pr2ua%rt5N;>ia{y!(fH}t5!E#=AM18bWs<@bqV4Rb!wHAJ+An4dxgJ>G5skqh*+*E>!DX_N z?$_OSsFVT7s*(IX;IFjiO9A0HGsi;4nG^UkEpuvm ziT%TYXmY;Ynv!2>q$+uXmMadBrAJW)WW&nKo=$xTzMc$;X`Ipm{wg-T);9cE8j1pz zva3r%oQ0yJun{2kNcz`Ea|?%Cm$XDiaX)|vKRR}+VsKO^Gh;BqJmUL1Au=7J0aNsb z?sT!)ueV<~)x)y&fmA8}XU7?#{-zwU>Spw>ewlBv6MoTwKztCoh!Cpy9t6$}Wy8mw&1M&rYq($v zQ(1<((nXlVuKz^~oW%u(nKP`>LoM6s0S!d_WHOhiyNv3pw=3Gk7ggY6N?$a|g)RT1Xk)zmRUd<*?ZBVgSZM0vM? zpLRq&_hn9va@@!%@Y*>3PHhA(K*{cm$4HfRcbQhdau=P@C6EE>P__Kb;4Kah8PkM=u?s`qICNRY_Q{{kBH;2wd~gDJ)15jT4)$7G`UuAhRX$aivR0+6QxK@zyEp%g=%?8qb;{b`;*#zsS?AER#+e@C(bMd`@o3(4)b% z!&D~zXRf3KG=mdLw2oI_FyQI75oeJo?6_8)fsM(kROSwt+ikI%+z*LP%;uYWudz5qUf=fVm ze9~IovN+5zdf3Hq5zhX#{>Up&4EJkY)f{ z=JIWrItU#HYLZ~-VBZGN7dF+;i#mUSyP6$1I~;q{D#!1ZFeA0CX@*nMTPea)7tszkp5bw%U33yNlV^*i<6sX@y}2dFK9 z1-3#q)U5W`fr)cE-`RerXLfYR@Q!HpXbV>pS{HxzQwc-;Hu1XlF zHTK9;F@27HRX={HatJYnu@4*#d-UDJe|7hK;9t6~`J_J3^Bj@(jPt)CGC=kjsQU(Z zpu|#0nKu~*6*+c6LH*)zu%1(LAZc-R26n_%q#Bm*>bNiztzr*k{SZ9vyFsq1FSNc? zGQ@Bnnn3JShoL3x^gOY_+N*J;dYF2)8Zix!`3mEiE6ZMK?uAOVx^TEdnBm7FU^6^q zGoHN}*>`I|@$VqKH}1|=ggFq}$#7|e;ncM2=c-?{F?I~kQY!-WN?A)+mHPd+7A`JB zDqv7lN~Uu(~%&|2ri$0cH;a1;?uQ2-PP@n#uWE9v{nc(O1dsVsOBajq_RVMULv5fwrOW z4iAjyJY)yAZ;Hphs0Ha~lSc*U zb&!oBiiHT6xYN;j-dov_58Uw}Id;Se{1k1Lx#)1146ZW2=cj71{J1g&ivTor7W@g5i*0?J&*-5w##`rssx6g~NUH zlAtvQv(S&~Ic$cX`2Z@4es@?%b1tZx?sB>IlQ7lx2F*oKac3 z%{oh9DVp8?mXlD=Wp8-vUo2@F6`jGObA}mg=<3GT-tY5hKLL{HWUBk@=RFgn=HN+U zLF+$PJ!W|j3hard%BAg8$G~|d9I|GW}(Q|q4N^6!ZQCM+%e6H#0=;w$2-$vEzm!AJOMD*XL&{ue2OcZUD^@-T7#y**laEd zv|zpKmA)hX6M61f%S8wOQmbuSH1@zij=w`A^mBp5d4BC{N_c5ch5L)2r~3zLlP4G{|o|ltFC0s)f$2 z<;#_rPioQ!i^F__sJ`#3r(VkxOxYg^W@U;MU%ZAcJ~WddTPh^gD_lOlr22rrIXi2R z3J}%-)48~XxP!6C8WE*RL$Ch;Q1kjQ&QuQ*OwkwsVV$)d7!8JQp2e$R1$Z3!J$`)5 z91YX+T#xC}3&8`k6f=Nq@)&Ch%|{HrBd!hS7mY+ZxCnS`KZF9{vkFB+zqa5gScONZ z>U4L!wDIP+pE;p8`d~U%R8rlyRIP4i)>V4tyT}ZJ$c*qW%{>DVCd5?a#bYJGBd<5_ zA!0O*_uBjbz%^J-Kb!fgG8E-}V3mnwr%Ii1%jaoR49}N-ovv^XeugkT4-UXz;k5B5v$A(WUr2@x| zX3CC6^|z%6Vtc#K!B&$p?%a0b-OAL$N|#NZ6%uwIF1c`f<}A@Qvb&?22jPL3l)aX- zf`CSn?v#--JtrfCDR2>AbS^WK4OP+COHkUq&Ba99$}o7I3cSdI@vny)dtebi1!Ljz zC*~Wu<$T{_$yY<-WO4d-EG3f+&%_T+WVbc%5xmVTZKzB3oh!N`XkG5;H{_T6nn<17 zl#T092C@Js*u>cy3jkLoTG^WQCCYKsx=qG*0|>d&&p69k_hEZ9Uv@vqJ1>{k&6)n` zzExo{n!4g3Gy65s5BsfFx*wgF(axR5Z~kpLu{@*ziuT1CftRw==ZYpj2&E5m=ruWNvD=C12K68|f5_hCBL=O{ z)*@ZfB^s*pB5C~P**rSdN6{@{-iu@;(XnhjrP8R2Ze~H0(l?Q6kdz)C74PThA@wVD-c!X0UORHZSUJ>Z` z4^a8+GUM$FMYQSdzt^_hHN#)NLhX?G)5)o6K9A|V#K}86;L~T zsQ_coa|}*~WjOHjhFJ%2QsGL79y>Q0HpGzEqPe4<-C8K%8_Qh z9uERQ5YO&Byk?2G6Bn9hcMk%v-*>Dkk?;e&Ajh+(rFH< zASZxBB9YVl(wk+sb7i%ZcC>M#u4Qo$C8kgJaMqw~ss*>48`rSV*W5-A;eyS+^?3OC zGS$spH+6Czh0<`qKz+Yei;O)lQAHz#Yy0lPc21~cx#T}kG)H5aP$ynjXZk6ioV=2ru1KNd?U z0BZR!&}HuA#$-~2kq%*a>D`p|($GJdf=0O`lYbc1Q%4Y_%d1=2qiV;a{jwSguyCjR z^#h+7xmF(l(J=1cXpYx!qY-6hMgV(`mR0D82=9B_rznsN$ox%6=3Kc08%J)&3TWis zv-p4a#eGV6ioa!FvMngwV&qQXm|p$Yi6n~i0l0%L4(d1v-o=uTGbDU zg4=Cp0*>Cb6BXhf}@z3wy|_05l0pI_RJ9Q@kJ-70vxT8D4Q zMdOaQ$V^PAFC97Uvcfy46mB1G_8r)bHdt&oP$%)xHz(df(ly^axN+c;%c)-wRkPVQ zX)F&CK0|^bCqnqV19MH3G`75lHEdxpwO4Ffiz6nU)Lz1KvO+Z^(BLGp`0!Vkd$6=w z?m;zj)h?0N`=m9lUGxM7N%iYyy)k+z*@)otxUIy8yH7$4$GPkLg{d63Hs*Dpilurs zq+JmMbYUat=lt^L_rzS~4o0P;HUZc;HH$NhC45GN*?-#frt-anG&h?i7PGPXE}Pn3 z{Igebzjk_Smd2jN3k!zMkNqD4EUTYsIZ zvm;hn>hD-`*9y);6AcY>YazD0eU8gcZ7U#D+OVt~4PF_+; zyb15!__-(Pc<#N*{V2$gSb5s!ICDHLEVa&p$hdu?^wdGk$bZ9=qUZJ9p4bx7F8G4cMgF%EARh1aoNfzykBymX07pHgr0$*5--0vkf8a_M-t zS^u!q(d?@t^EbdQoPLHwOK_H?uivOSP)lBTYV?lWhB=1olqty6Su*+vAtZa6U&nm+ zFvj@~o^*_Y85|p9y`YzZe>B)_0m@2|uiJiU@x~!@-zM=6ZD1#h=dj z^h#}=eE%SK)6YZW>niWqbyyNeu$BKZ<~7`U-Og?2%y0N|#Zo^nJ~c*%6LQtH|51^r z_^r`0R26Jj`l!o#>omK#{X^E|UN0o1utaLB#Y(c`hde~jjw)qdA9df_~{3yx?X5)Ic4Z9ih&MDVTa{N8ef)P)lHSucp?2>PK!uTiCQgNyL6ZyjhL=N`!t}#_OOQZ1v*Aoua3u~?N` z>ph)Eov>E@oA6k3kJ{yIz5qg%63#-6lfCf*z;M4TV*IytOuDtp3vcD{G^)?mM2>uSwq2cXLgd#+TI2Htddnj`PeY#4%TnT=N8?D zr$EA@==L$Um8lwX`q&#&@)q|tE3Csx0Gw5Tfc^HF3an7Hi{l6TB7yP3^j(`2{YAIr;VmH#~-9@BiDXu-cBT(qbMOVsEyEsYyr zy5p(odJbG7i^);ko2{EphOJxHO}~-Ny2{eDY))^Mt#O;J6%?1_m44hlZ!8=l3~FbN zh4bkX!-96S)I3OfmmulC8#x*X19)$PDQ}Nxh#$`*l&VCyHuRjsirg?Gg{g?=3LQi* zf`0=6Axu{doQ%SEALoQ`p{Ja|N1vebiI}wW*O0Wh%G^=e$a#AyR*aFJqTY0BgT*DA}clXa%cZCIRE4uxT zn$8(X?ai&E(r)EN*w;U%%1XMbT#kL_F3Zq-t1GTru??gh=^hCa-`FXbbJLD0U^_P1U_DRv zPjb!slH6c@Wv7D5IdX~z|xn%g}O{}aPi%??bKZd8*6C{{3AI)yfIF4bVZG(YQ(!3g6 zw)IUZqo1ZS?uktW|86$gigHb2x1~A$r7fyIHT-6qXkiWZ6H}Q>Oy@@nxB>7Xg67*Y z>o-rRQ~y)J9TIR|5bi|Gt*bJ4tvn@UZ%OV(Q8l25Y@ji%9wydqBDk1WJ_~3x8^UWv z$`!xwFo1-3~zYKxWC}S%=|+3E@Y3 zuzf*=@)dMLpAzp8TPxc+xWEQ1Q(~n4M%lzHnc3}<2ZBDI9 z2)X?XzK@U(55KKd%a9(Dzjf0N3x|*}KO&H9lXjtc#cs^E#-~%wOPGGd3P*>_4ha$BVi9{Dj2BLp&Zd)pYKy zWIuofVE!Z$4Qb33WuOG;*|@*mC6e9I-&C%B}6H!brX^t`}atc0rUC-^LU=wC?8 zPK(Aa$aOh;Gd0lEGj;=${dW9jFYr?iZ_E>&0UFlcGj$KhZVr+NGURfZGZJ(eOr28# z_AC*@L!k~5VmWW7=O&9p*@3|Zs%9+VEgTyQ zmRdBz8e`r24$y9K)V&+$S5_tEHHuJ}#!bh$=+|sLmfN4r3w^X9=S*lbeb;$oa1<)@ zJq9}q_M`T)iy;xi!!=D21=xo64Fo=a=d4Oa?gr)y07Ja4n%?LYL`2ntt5`5`un!6* zm$w{Dq>q1rLw$SOQ+?(VhWi}^wXOt@qYzOwv5+Gt@0O?tpQpl|O--4A0j~;9;_qmC)ZBzW zWp%gD?Ui)QE0L#m1YieMEe|&cYe#r25{=J1uiUbRdAQSr+Iv9$L{OO^i{zvfp%Zg! z^Q5H}Ws)V`;Qpo9dvanDTxR|cw3dYx6fC+q_?O^}*0isVpMZ5H124ZR2oZh@`4di+ znu{colT>JiZuBw!hLY-Qq_A8!EqNyeHKll6Qv^oQ-D13X<`?3G41yFqbIoH~WVH6n zcp&AhHoA(*S0N%YVl+YR-pWvr<&wL;4m>3_&~6wDQ?6-TamzB1w(4l4bbVqE>ZHit zwsK4%fZ*$wL(GrjC?U~AZ~MpUwrS3!MKgl#@`nwlm;wky2vgVEYxM1=8q>%c>-AESK+0Q#vVgM4X z#vS@ZZ&XxT-w%m(E;m}haXvNiKcl+8emw{ujla7x8Lu0Ud~48_)S*K(WBYilYuc`V zPtF+e8aaAoNU$W_;Ej>@+?L5hrjKMF|Tg3s4rmgZCzz| z6@;z|CQt%rkvaJvW6X31n+8dB>1redQjN73X*Ia}jI)Q-fW&&?bJM)@2%!$nb?jT+ z)bi`j2Jsq3$#@QKxVo}b!33pJTuEX1N~yH+RdhJjwu-3J65H14PTiqJ2ke>&Don4jU>K1s z3)0cd{DGh$i1^)L`<|%4Qoa}iHGx98ecy5(zirq24E8f5&-N0%Vl__~*QpV`{kiMe z_QK%rU;h!fbVoH#20>d#7Nfz-T?rk3Yo8={_=P!5alX;6la7J1L+O}nLiO$#F7L}0 znO=$#F}wyTY!7GXb~G6z+W5y@e$+u1k=-@!{@2QzAFbVV)Zh(E&(&8Ats0hu&&9|6 z6T3Dixw`v+2gJczq((WQo1qaBx)BZ9c+g?XuUj6rzEt$mdDb|3y?zu3FXjzz^}+CW z#?4S6Jcft_&Bmn$gVhsn_d*5;gUpQNlZ+HJM#xd*&auMpRkI=qa9?LUk|R@= zBaZPLVmu3JAICI)ktO8>depU82Dx=h^14XTo`;@|5^Q^pthU_!Kndve^m5E-rA!#W{9;KOV#eV2!TP6Axp1 zXw%t}fhMF!n=ZNsOaLuRx7@7;6YdV-nL10bsd63*x=*S(h`aV{Wrd9Xte>U%>W`eC z4cNk=Pm3e`d#_AW2JS*+RSdXZl>j26&DPS-!lwC_HUDAI|A{rcoStX>_w_zP+41oq zD-(p#i5g(jUkqDnZCQSc(_mcE*}MXvKWDv@jq0u`6Wfk-L^X2P%_UpMh!?ROF(4-qKx*K5&P77V=X`1f`och#%S;Vytz`d?A)SNvtJJPE*Qk>+ zBgWZ*yQ_yh1Yh(QA>u}t?+LdLf1TWz8OB9`)f^`lEwQaFGz_C60nZK@V|R!#x(H2F zI&F$eJExFvsz~B9Sl*_3wLu+J%D*3Kbj3~P9gbfp2zeA;5)Y|6@d+k;ItIg{%|yL$ zWiGI0G}gSq;NGHRO2`hbBso+BwX9aLdXfXhjFEsy|DqjCoo3vAoqYFUvTONKTFm*( zZj5|;WhJ`aFj&IA)M{>WR2I*>kq;nrKswB@(=6f=Iy)jX0hCOc z9QC@6oQS$e>Rk-ND{kD9H?fYG$6-a>A>vd#hAivkT+_SpPF~e?60=|xPou0hF=2X> zuxp>~b+;4un0dtr+{!yr*fOOLV1=`WG`)92iQ6)ncW;B&>6qUXXJJbk;QaUo`wS$#WB+T~oq?O8Q!+L- z0wWC13GWZ@E;j7omthNVscZgp__Sx3s!2Y=m1y^5YP#%%pU z2Y!WXpn;^v$Bh)>rrP9}5MT>yGk%T#h+7VIYlAZI`;CA(cEv3!NL)x&blb>J&3rMA zSWFqf$?hTKLXS@%izNXEbKSMtS**eV$~ME7w56FgLfdCh+U19?r7Af>6fkweStOi zw!y@26^MUCykj4;kxPx~_sHz7+3IX-Z&U^5(KcshnXziCC0d#|RlC*6Oc{L&kf@CV zp7?KzS2R9}i@hYgmcLai2BI?+|B`vAlyzr7%v&=v;`?{*tzm|SH^(b216PZ#K2A2e z;Vv#Sz6SWxL)U(ztnNM<Muc2SE51MPy(R{(i zJSAYN%>LHo3olU)`P5+rg>p$`OB;JWC+r%Dlc@cDH(&PM z;LjPyVC^Mc>_%PRdy;XVSOugJYdqz+%=@RZ4{Nh_(*ofNjp&z`V{A{cGp}i4G6y9d z75G)T9(Yi%=3f3IY|vQ>N@1e*Z#x{^mi_--01FRg9vc=3U1v_4qvj813@odf%?`g+ ziC?tTs5P|e$}K7IUlCUxF}s1mekhvUlNT4CuS9mpultDyN#?cb7XDP1C#1!4J$LFD zMRyI}TuIXRJ2;ZDy1Hx=I~}AR3YaG2k=X!hIWPLG&~N2v?zJ@-w})?jw?f(utDkt_ z0B#A6PvVA13kAC6VI9dYUqImfG7lPi25X`>m9Fpk+ICRbSU=ogzWKp+_-Ge<=))#8 zyWXYhypIw`;%+Qwdf5123)cF7gYET}!i38uuC!)SCwD$`m)g;T1DfBQf=eS!1bWcm zMbqv)>zTNSv~z9iJ_S(;N`&N}rS}6=eHHx$1H=j@OZIOa3hifpT`4898&L7$zESa7 zHK`i%hzQEh?3iD`tG)`OJs@xD?&&W%bm8d%2vdz0<`HwaH;afw0~PJAAPsI#~?sLhY&KmfK`7yby@TWNQcDZgif}@lWuG3 zSDQkO`G(MHo5Ii24JjB0Ry*4^VSRTGQS!6&!^Qg6qpn>9k!ISxHQ`LJhPT_;WSFp(se+?%%>W4)7h=e7x-d03BLo)raUX792|t+$bLI$kO)RBYHQOy%vd z&PQBC1OU;!2RFF6ozsmyhibr_f`0me4w1#>6chdF0%7BzOjZ17er&uWCg=mBmG;0` zQ0eEzhBOgYu0^OY{satB!>;-EpY`!Ua_^_RSqolkb)mrCc4xyo1O5uH&L7747>@!T z@d*k$IOFt!`%-gHXF}Cj53x&3sYqVSu9J_e9w4=wEcP4iUU3Ie@dFsGydWy z2ak91*VI3qmdNfAEKgC9)W^E$U-NFo)~-*4EU^5BE3(z-rA=Mh0;J3Sxbdc3lZk!3 zaliiMETlIvgd`@2ur6C`G8}z(5~h4tDL*TY=g%|QWtwf?3P-=-Aq+aJ^oDNoHqco9 z++Pm{=l(g>oXOnxO`9KFcdxKCa^n8{mg@+{{M!=&B*+K%U#A6*6d2W}mVF|)k&sob z?z-u01Ag^F&D{+F06bAK?=i{Gn>}Gz&}uvfJoYwEb~drLPlz(g39i2JLHjoCG-mIa zA{iSZLl`?!LOSu!#yo%~J}2$>O9=k!JR4`h`&!71A)baV4Fin0=ge1+bz5l=(62So)oViW%=!lmXXx|U z_9tHXr!2gj5G9yoGG5wEym4K>bMlCF887HMbkESagdb-1_-5etiG+2N^~cO6PtDu^ zqv@-{>S&s72X}XO*Wm8K9Rei5A;H~!2MH3~-915r>&79tTW}H_f;;^4e$P1<`*P28 zcU4zcueGX%R51UsqJi(1hEQbTGiLGWG?GomBaeuZCy9IDO@$du?a_^}m~bWSQyTQL%?w05$>dl-bGwD=S6RsPGhwJN!DL7bN#9G~! z6?~6|Ikf^ZKWG@<<`%eXv#0&qu%@&nkmY(*n^6%9}SHngBL3H5h6e%-U_s6gW zV#buWes;8RZSLu*;6$mI;R92P@Z$zZ-%yPxe{_5ri@5v=W<2uri4BR`^2UMuZy%2( zH~rY1KKzJTJC(3)bl!DQ9zsp`MG!8hrX zn;G;>SHN`c$@Q2C84VME+(Vx5Xh3uohJI6~e_viW%meLS{Q*4Nt zq~7}kIz|(V}=G z*9)ZQj?NqYb>m0UcJ#c#`}T25ndRcz(*$92`qlMEY7pHVO$}O)xw$V)cDMos1R7hf zKfM;rTy@`*1vxd|Ua5M;*~#;|coETtGG}3DhdCt-P!N8Tb{?*0ucybRZ&9h0%7&Nu z?ZJ~|$-c90yg|h)g*tXP`u-Qbjq*Ck9&TNi`{kB{3dtL z(3*j)+Uulo;v{jLMf45?;kulXJXK`eJwtfe8VP2DD{6M8DA+w~z*PL3>kJ=E7G^qe zS!ggJE5acPjR!Pu^_6gHLQw_XpG_5Wh=P^D?DSRebeSPZFxxb&d~V-;sB<{zL6VZ| zC7kVc=PMWI(AtLaBvFtgjZFW_&c@C~+-KA2Oh3&v?Stw22hO#r=XrvNJP|r6O2n_J zy*tix?PqsP$WnsxqHazJwR%*KYjM0U=gcS3bMU!H&;y}1OPd1iHvLA_sL@f^eGn5{wAK340-c&zWVtOV!qU47sd{c4X%IeFrm}tkVvp>45 z6?I6QBxn$^e72*kr!s_>E<(vq3zWm31}in@i}fK*Ff+IrcGqWio$fGDIv^V}l;Fi_ zr918HRe^gt;Gt->z8QPIC~enGF5>PO`j~!s3h!TgtJWfy`YfX|+Z0nUijADChW?nO|%3$61+e{Es@_|91fCZr_Q=T6d9 zDccR_BF_({a!F`p4}UQa%3)#@##)iradLLg>uqz1 zQ>a{u)Q;l=Z-rlMYNP%c0q(+~XERU^OP!+t2vI05+7G_G8gl})7_`Xy?ng%d{Tvl) zb7R1DIWZb_9zmL|ZA`z7-d6-?^!MLh%W>bq-Tf}&$R3N@XZQCPu-lCH>%GG9d=$SG zs!XSB;Vnz$YSTxI@RJ-S5WKWl0e|y5IdLSq$5LKXzAF}QuR!~?$?&TkVk!v7{MHt?IC?!`Se-nb1@DQ|! zxFmY!^1jMi!ZpG+DeH728!=3(m>#C79YGTwdg5eKMXUH3i+I!dN!=Ln`DEjRHoBGU zrj@;0FkHzFsioA)pD~3#t0N@YQ(A7ZDwEwleg zxZR1ozBKhxI)Wt@4WzxG3=rA> z1^+R2#zfuq=}DWzEaqjF^eh1wR>i0`;bh_&Rw{FXsnr!=x|%Mw%0EII%~zU?x9o2A zN8)^SMr=xX!Zn=~`m+5K?rf)&t0w-p*szd2{RO27th7py@vxFgkCrXgvrrOlskIHt z5AFC`c|R_oUdIO>LdS*rg*vIg_MTA6yGUJC<|UCRUtKxXvis)lw&mbEo7S=+&VP^o z?6vG^_j<&wcJmyed~W!mj+MS3g#5}oWuY;UNYCo}Sbn~!8gJwx?l+I}yuq)C;@vS5 za{PIiALJ$IH-#LL$GZH<$EXlVwRaD*PT|o|FwVPl!nwx2E86acj2c3& zwALQ~g~+K-f4Cit3i0Bq{IID`jA%lvvSWKs>n(Z?+s_Ajuo#xJhH^uqKc1r2>f3)n zt#G?}(lgFmjLdkf{aLkg=CzWfFem&qhlGEbJXX>yK-ivvfybve0Yh49q9dFkNu@pDp|B#Fp#k%R4~}kDlf)UQu{_5!mOy{|?of@9`+z zkO;0LUZ$VJ-24&;*J4*Fg{hBiZRWiU{N{0xwFY2swboBH;VPtU`WE<6VhDudMY?e& z&aE#ph}2n+i!W)md_b%b>KR7s77O$8;<6$`g$1K>Xu0DW+?lTEl3nFieT#1QO`ozm3|Pg*&#xm`KMd9Sfr5noNu|UOh#Lr4Qmi&&@_6O|$>W zhej|7vO^Dp&(fPJTfBG2{Nyfn6@z-CbnA`X4csRS;#3su&^EhOp{}>?C4`2alb_e4 z`_4)Jt_Hnlku!!RzQ8WQ6-J{Bn;{=<2=H6v@$WcuAnDi+Mdh514EugInCoi$Knhu4 z)>-io9NC=Dt9#>`;!gyOrCsE$J}T6}O9|eL3z6qV@=o6l{P=Tq{6XD1-eok0`|7Z- z;iJ(%3gf3gBqUp|HMfXmXVPXrq6eF$C`GFJUE76xX0ju!mAVcKK)y%-lXx za3WC}c#yGX{`%w^jdy6zu^wIWG0$q{?d?h2E*e)#58chEax@lfK9L{C%l$DXq+5Cx zf~iU zNv-e;N~MiE3ZAeXBSUTzB8>0XPhR2NVgBJ=m->jb7BXcchY=9ET_8&-7Dj1EwS}r4 z&m(M{zr2vwVAE_*dtep*_>$|dpvwxTD8|rm#5!7F-4TWvo|XZ|xu_kkA+eHP^IklcFokWma=rIN z@N+~~I&^a3tAFrG8-bXWOTu=e`i41U7HfyY=H)f%W-DA7(kb<`P2Pi1OmSx+<32iI zOX2t~a!lftb2{v;-Km+Co0|~2XPUzhndr+ot-ckQzM+tdU_9>}Smy^72=#=?)F=#cc=4R9uRW7C5$fo~3z3g> zw4HP>1^Eij%TFD~yIfMhT=$K58^2lKU40u2%^Q{YLp5> zfi!=jg1G(a@_w)-CijeQDGnUCnuC66<~5T?RH|ix!|QDgeR5ttxYcJBUxn2~hVNYF z`tMN~z;DlW?tO$^s|zte-Bgq1fJ$Tf)bld|)9_SrDb0ECw+ex(fVI6#-=NqHmP!uj zN6|E%6CTGdI@6!929sCL^C&v(BfVAE)y z%)Gcz;QPMT*l`Y`eJg`C1^iA*s@EV?@KSMx6uj@h^_}C(e7J4-z3qCg1fh{xnP22} zFkY{Rk7+_A>DZFv0UdeZ&|*W|0Xzr68^P|a-#39*6d=S{YSI4hO}hy^R36|s#bix^ z4`~0#3=DObzq{fJ`F8di4*v!Yk*;k6s2oqIgfsBj*U1pumJ^wLeS&Al1baIH)c4z? z1rf35M*;6Yrsg95XT>Wb`h1jV$1Vswj1+PmWJU=Y_H;6%yHmYcQoJ*=q zyf6eM-5=QU4uw8Qo&O7(!sGQOMh|$-Jqm!S$1>9A61Ob7^tyR;6?i3i{jE$(C+WinnJt z#%_{=ssaenqe2QgGn`(BkZHn;4M&r%o>Lk^`j*sPgc4oaxN+bdopgvXrg&*^T<0J-azm_EFj+N^}}wyCn?NfBD5QQy~x^SgZHoev4EZ zg3H8X#$ql#f*#@`0(i*Knh$9duGlj_;ULT_SYAZ*+2l=PcEnZwRjyQ<2}{!+_@Nt- z*@yt0u#4hnEpgu{sh+pmjsy(gpbiDq)BxSXg#!@#YRY1si-eusuaSq)i*0pw9H_?V zBt(9@lq2BZ_G(la!^uQ6-FYzbLmNB!2M2J&FMq=eF@6sB<*vMW*-g^zYf22JvKA;p znzHhD6Uv1W0%BOy``GaP8X4;R!kTqR?W0YjyV&8fdwrcw1!2=f)PWD|d1tc3t?-^G zIFRY*47=mzQ0x&7ZSgg^=U~s;AVKlMYkxAx4psU`Knxb!oj4plTtqm`_VT7$|I@|S zJ~JJss=nrM$)c6?q1+ueC2BJ{z~CM51`*-G^Hc2W^9){|7y*z67|z+fJf#vM@!2vL?+3ybyX>)){l2z}LL@L`Wek-4qwW zU*c{$)xiD}TmA3X{n zo})wct%7_;JDVFk$52z+M}Qog>n zqE{6-w!jh=hSoZMI#lse{K+USTbmkmye_UMsq;-=I5$m z^cXo(3T?}%MbR9h>PTKB%RTRZw=Jhw|5%ONPCR@cI7#y7Ojf{M*k|hV=RR$@ljv6> zaAqI6bbe?>1{!5-y~c^TmSZ^iRdmHI#LKob!w0{C^ZqI@B$zG?L~{`+NT|}SbzuR3 zK%QZoeAcRjIXM?KHF1V8<>&<1bUF8db?}cW=EK0Ny>{X3zo2I)C{tQ>Plt;JZD0Cw z!OPQ!4iwNll`%GV%yBiNzV8bhyke=y#}9oSLx>(Y;72MoqQD8z8c+7I<6nA zK*|Vv5)(OTusobBmG9kaxW(L;LUJtZ2uMQ4PM!bFkoV#5v?;ne4TbhyURLQaaLVTs zziZw&{Z1O@+4GE`!c2oZLaR_AiOGIh0MH>Tx+Ja; zJ_o;E9=0R?gM;2fyIwWd*;?+FN^#t|sr`7#-Mq>lGwe;stitA6tI1AW9uKb($5E<8|GFwYY_#KOjwxTFa$BKcWtu$7W|7t+<2jIzd-q#>-&4^I~Tt1 zXT=u{Yi;d{A-lC*z>@y^G=J9eW8V5`dN9|lt^~|KT!<%{JPOPNg;~E{o=n$oxED*KOsc2jUv zC_k4fP*B}~qsFhdRE2r_O&o^Lmq1s9Av)5RSO=ZEM6M^>7Q!+BSZL$>kBj3+KuC$|^rN1ra}pEQwC#|~Urs!6=r8pYI5F|+UphK@8n%SoAT^$K^T2n=x_ zCSSvvh9Qk3v4g(|@y}k}uHF+B$!q75uo#&qth2$OyqB**kZxc^Y{m!z_B?aL$^|0+b0H{YY$f0G5-6 z{SJpFyc7u4z8DqNdUP+oxX==7!K>@9CZz65)F~w;{z8yKD&!lIS&htXX}mAKLSop% z;)%}i_U^^alX>?1j)(A3*~NQ@C7HZDc$V_6xh~b047E9}$k|W$-zf~SdE+oiViaf5 zOhWvL(I552P)Y-lBk_)|=TBg29R?93cmQ#|N+9BNXW3l}gAG>mULS_Lz6Pe$`t#Y> zW-(U~Ga&#rV1*E<*K&GV=`31+O>edLB_XW997~0@NQ;SloRF3I{&JRYNBsdGYGvH; zuG$^9fFCX;ar@HuuE#>=!eeI)Ob4Mbx~wQOr-LQe<3U?A&mbMh+6F!}`?J*gl&(8T z&X;t#$(w0}Lg*_-3(m7PlG!?zS7H6VJ46gIP`DK*fA`^_-ZeiW_RG&NYiIH}pYkzr zU0XjCT08Kwj)b=WH>S4uO@5G1x;L0a(834rYB{vmCPs;UO z0wVabT)v18)%6ZMKDdQUH%UK-zYpAFO z7c^q2f11!Djvs8h=;oXJGVSo_*)wQ(Ft(QnQCj{p{MmH;P@Vj2i$w>FD3tZpL+=#3N^gouXoyU1p3=3OmV(g(MHjN zhTiEzTG%-%Ys!BrSKzfbof+IdV0rCp3~|IzY?j5Ly-7YPUa6IM>@H9;JAQ4xOmtgU zq}va9rH1vEO@zbu|8FPt0Rpr3pJREyz$+*yt4B<)uPY2T@>)>0hhpLdlbM0feCE1!K6) zCPHT3w=u>!9!mi&0~*aca;O7;1P)ySjJv=%IU%2wfFgY&A=$TW-IGum!sCmdd8I)S z&BWM^9Fi$~dhK=iE3-E6o-+IQ+WCpw^T2u4C!yAM*yqx{!q>40&GO>TJoQsC3YQ1( z-;ZlC)=iHZ@O6dyZ5 zHx}$ptG~{(cNp|+Nh)TngIPI^CBlWei$+T%mtyrOg8K&czDyD4cwnYxH>+N3pVbPV zU-}{aJ6*GLebSoz|5|`P+w)h9@bO3!Nr~v&*G9~N$Ku;~l5_#|*PNHHkK%q@LfSr% z6+bBgGNh7>07)ZGBWC>U`j>-l2Kv|e-qH0}s~sZniev*{8~mBKG>C8P%?sA7b1&I< zH8~{&aa8C-unR1l4xmewIFdAF`C)O{+kWO!lkK;cmRQot45L4f6J1kK;9UE|Gj?H9 zF-{)WdUTA_ao8x!iPcH!6dKp0KiT;MeD~edwzafPA4?_1(MfK(?$l@Z!WD8+S)iXZ zg?Q#$#?viW(*IzLf)Ml;05%ac6Jb)6gD;niVuZ50osZeW`~`dmW%^Oj*|$7?Jk+Y@ zhW;;Y3MiubDwXcB4nX?IG`9L47mMndg9m?ngPuCY?IhhxDotPv^ZnUutr*3rI-g|E zEPCgTGm6{OHfE2V@|b$3#=)6g{bGS;ddN(NAAKwdsJutiKFHE%1}#jF}jqOijebvp$v(HLEzf+*Btj^aUDa}a5Gj}%Hrm? z`;Y`S`#EhqUjK9R<7EB(cZdF6f-3#>>!zjP_oM9YBj67Y$;{yKO3X-ucNN=SQu+!E|AN&DTOhyERG`R{ssgNC z8uAykIf@i|G(M;@Jz1@|Dy=ngcaDe>-&M4z8C3)P$PAOZV zWqzr1vu=O^|H}{7lHjH^G8qUOBM%ZZCs zU(4|H_*9kgLbE)lAb%5R5&_!1usG+n=cJD+{;J(6?l4ukLFo5w4bGfx`C2l~k=sRn zgsWuF^Z{;AoTIVN+a5!MeYYvu( z;ddEdX1>~J{A2<0(3gZ&?wcUfypqjtQUkjqmkRix%K>xwcc@UhVu-Uj z%!6533qJYZ+`H;=)hBB3kZX34Mx&aP0c0su98`-UGN1(BmvI?<&~rdRI!?Eiag#D%hNQGfUbtJZIY` zB<^R#>S2Z^_8LfUsJ1SR5-yby+tHJjjk8+N2oc zXtBU>p0{p3Ioevd;h~A*PJ>oEz^?UrPdZr8S8+2?<{;!H^pnD!AEKWzajyu~H{(P@oaW|PXY-6 z{*;T}k-CV5M}gKzycc`PCG_^b09l|=n+60&NyMX64FZ;w-~~RB=7_@x*IkVyhYJ!G zc~dzh(jy(Eoqbt5EM75Ozy0u;69T3C)wY=>rHuansz2RmMb}zb8Bil500d#4PwbB$PwTXIm;1n2JrSLWdzNxSj_? zhw9uT_cOU4Sv|CG1(7?B1Z&$-hG>W@LcrHRm&MG9KbtsF0(TA9SIYtOhxnIL(g#hp z*r4TW513=ZHiKus?x0}kbKwudVl-ej{oFl#=}VmmeEEr!-|$Vcy$&DOjJ-Q443YX@ zURWXbz-5&UQk{Xpo~dtw2&nXhAzec?+H$Et`mzJ(b<5~MZ!Fic_>u6-z>IZY7uC0N zf#ttqSL&x{H$q20kL#PG%%rQn+VopjR>_#MLKs{-pW#6G2eM!Y*q)63aA8chxQhm2 z{i2lZnI+VM;P~V6sFMT}B1hGU0MY@ItfEl1=}ck}=TK-A?x-WiAn>2kP^i89ZsEMN z-wN#gopkq8t;#=Jer5a`BdZ{VL-40HBuEo(Q`m%alcgR!DisNc*I*h+VbWB$v;fSU z#_pMRw}IF^X92NoG65& zwC+h4*_$+wHeN~aPYf_+(R^pg_>e)+z)KpA>IZ1 z5;UM>1YMTu)=WIsY02(jl4U`bH#=R%#}6K|-I-F(5_!BZOfkNGhE_3zacCXA&)2kO zAB>g?i=1!5T6As8mq|=hX9_9pAY{{s~mdoG4FODn4g9HP!>v1u`+vo#G zT^E~cEE%~6I}A1nf%w6o>Xl;QUKaU^w=xLko|a3@L&=+u)rU#=(P1<{qV3csblp(B zpIhKn&=X;O_wW`NvO%;!k}uLATAvEr$@N&>-yph7ckw5qmJ&MJFVd%DfaPZKV&xC{ z?dVasxyVBzzTL7jh4>%410oh@{R+a-`b1(1D7lujQAMfl@bJmkHPO>% z1K`+7^7}zq=?!+|{J<>Nv1^Zze==9~I8&%WKr5u#efQ!45jZ>^>_EpTjgH8+q>Z&A z=V~gf4SnfUO7C&J9yurfIAC*lp1jYOA$A~S&L{7#QM~v<^>^zvzo2A2O2?0_T%eEV z#c+@b8zB*$WzG$xZ=OA9^W8mRy;oP00!sS+yThVv3Z$;h2dTvK@Mho@{8*NWgZ&>K zo$-S3*E_bcA=Fn47aj_2FvE^`JslKi0d0 z%$_+@?$I}u=EkZp3YqZgAEEL5fJD)9dLcoJ|A%alR*;Et3akwICZPWo8(Z>IoBfL% zY7dfCdVU1h=O#Q8!F z7BbCIhxjrTeBbx$9H89g_wrtPLN2~f=YX(1t1bDR2URDdIEccplY-gr;o+vetI0|! zezGDT{m6d86S!*?#Em=hx}TBA-Z33b8!BuFi2Y*N$yLuR?Uu?(8tg3XUn0Fr0WzuxkjYGUQBu)FpqkGE%P zzEhcOP)|P>%;<c|-ggR?0UG$Fydyq&^@X8mveG(D2!+SaGHkD^UDfl=Modo?&-VF{ zPg=9&H#d)e!`MKU;@q+N&f-{{n(ZPITZwnozr~9nWo$1ccig=~y6qt(O#`eLU~cMX zO6yf366KEW!8^bF`gQH$Je}Jd3@JwAOdZct%)j5qIXric1>AY2@w&Z_Ha1I2LpcG} z6@z~3XzrtJ(}jC+m}eq))8&n2_!#i+)H3yfP%&jZD33CH3_llp_@1{U#nWCAs)#7& zgqS=yysycG#4)rFOFfz#!rUm$=(HP_LSg(7gr;iE9UA&(6rJdDadTo%BRii1>~Nq) zq4D`B7zi=oSMp?r83C#pdtK#VQBDoSEm4^4pLciRq=zoj-{1EBSmyD;ds$LX>(e^x z>TZ+5-_PB=4Y$^W9=d6c(gD&KM`ua6IX7jtfMi`TcK!4Q(RElpF5vJRE#w)|(zblwFCld*ZZaHDTVtPd3)hOamKBM8tGz0(dsZ)cz zK+`n(l?Y6;3FUf>yio4t^Iu=x`Dqo{B(An>LBtEZ7(ZfI>I$mH6kFy(?eYShX{MAF z4u3$C+rqBEYh#>mPv-mF{gG!ktT)(j?2<|P9-lr`2zY{n5)~0DzBC7TRp9`tVTuHyb6WCON6H&XJ930Ks8h7)K3i2^n5}9kV1a{ zZGrO(KG|khP%*VgloNvUhl)xM(?5uV@%hD;^r~4bNeWvkvjuR@b1gMpApelY`(8c{ z_FLauHML;vzC78USY@(*C*7|&GI~f4j9s`_U^Ee(5|QfRK>5sY+7_&j;QX?AQuE!c zaXKaN;*Y-clr@Rxs=;hWzNS%{(X32-q=wW4g7g z037e{Gp6z-XyuCS3D~%}Mbxq~;BKVZl?C$ZjvYJsa1NCGP10t6iJIsa{dc?+8+sDV zDFu!`#3&vLI6!`m1bFeo2A>OWY56W&+CgUDDqYdlho%clU_+V56++`R;5=M{@OZ$v zk^!3G!bg0tlr4;5aTtFA;E*)u&Yj;v9wHyoVCNJM8o2+{&jLr@&hh)JYPY8Z!}+ZU zY%Qlxa;aB_06ecPMQ384iaJ~i3pNoZJ`DqCp8zrz}A_Ayp3{8G~ z?EOMt4F8{ZOCbG29X-7`aM_;?3qldD3V>OEs*iG%L*{X8qwpUqzMXrmN12d+_;H=F z_V?GWnkkDNa{;AbiQLF8$`F|C`Ts^S#n%))s0y{lveyow7yf zheC_pPycZJ=O6H)Z0RdGO{)m5OeTyWDn#UwpTRinBiB~R7jozU?s=~tfol5|nNt|8 zh&&=V`>09D0?I{JkM!wl*o+Po@_}on@*%i?Uq{aI-bDH`>jr*&_yKU*d`yyw1;4li zy7@Ausl=$`Of9i0=^)iUx#`gh<5PXO1d<^ES@_kFQz~etqCZPGprTi!v9 zS>^+1A48xTkWw1~b$ygwGz4!D*9&=Pgtb0f#G)b6?7J^^dxuMLReZR{AouXl%~E+V z1SRHQKwcVlo+CNR61LcLFGaCO@x!EdkYOnzRB~k@Kc1oZe_<_ZT7)}7SvwmQ%P0$$ zW;e(@wx;9dEkotu{w%cNASd3J;nHHj1}SYI7+lrT75B)qFE|-H;3nlh<|tDLoNWP+ zML;qO*X?PB<3>bgSSiS`oDSvNpLpyG6v*OxPY@B>in0mR3d+U?dx(VXZ3vGysI=X= z?q9b5+SIz!dYSffUM?4IYw_IVpjjif_96)w^Yg`e`GF#!-99L-dDn5=IkS)@O<-)#;SsTF*Yg>cu}DKf(#vI~A{BhrzzT9p9k~>4a+O#MsI!nFjiMMUUh% zCn>yG=`ZUYy69BGd^O1R!&fc7{wK`(6RE22pf|V{2SoVgfjziv z33-R8{V5jE)CRZ zOZC*{e;47$mB4Iy9oQXB1>D2m5*O2W*kJ$Ow84U+j0~|0Do}@MhqkYSeW_3)ySXzW zntc3aWNN>Y6vi-gL9hFYTQ34P^?xfq54<3PDG(DfRCcj6-%en1A;wa|V8)_nMdO` zdH|?YOODkjP}>h!70a(uxb$Ebjrjnj6D*;5v%nTMi~=Wz<9&F|qsvPQk3x#!N+daQ z2_sf~RrRMjqnin@peAya&P3B;%m&dCV^e8}q6hLmr&IW(J2d{!%MHU)L^9J- z6%c4f@~O1){9lZXCTpv9oNnOSVyK4xBKL_oB8K#@6#Y?fb86nNDf3Y*DKtiElR0ms zE67yKZZbWtf1NigSV7OsiNk)ZlgXWPbWuDYJvy+FglpjiPT?tVXh63I*K8_tO1i{} zNc^-Vn(bQgZ%Q1SbH~!J29?eW(sYGzPf5S`;4y!9*N07U<&auHQpDZ7_=iXM4^?F* zTipeBr!u?5Xv8s00l^xw4KlZ7_0YXY888N-+&}gDLL;#M3)?Jk+z3@V+2yJvHvJ_! z_tbHU1Gac?&IWM#8o3bzaDsn4{*}&JSE3y1g`@@-G5g)Bm9E9ep33jFOer_{yA83` zwjb+eX$763q%iV{Qhg;gk}vgpxB)xZpwbkJNpT`j;Lrg81ltv;Itk9G@p?@sh*Rn_ zhdsk*Vitbq*jH4O3JOK>zgoKoK1gb2dP|~IB(s|!s^E0W=$B|(^xTydygRhK_~rGB zOr`d4UtgC@wgdCIb2aI&H{?TEf2~TJk82bf)LmwH9caZ*mP`Jh^+f>vFVmdq*qhbe zOhsOS`(hBGmrt`3@*S6I1heW%sj(dc8q%7V_qf{FCj$5q*UZp#y#Ip6ZH$irW6e|F z#R*pr!;H${quOYgaLg?w3cW1nF>2>X(afnGGR;*}teIjz3dpvFcA!D-EwxxsDBFc8 zjOxAPAc}wHHQ4GtarxbZ7kHUiO16|Y*1?A>?Nn2MYsXQ#R@`xrK6}YSB6fjQb{O?< zzCfc~AXpo^|K0bz9ttF++sRBuzR%6R;G@}X1X}Ut<7t*#`f#QQV9HpxE#wNHLvX0kKAgYI;kmrx0n&wgAgQ8&z3M;GkPk@bjFU*3Lqfo5G?*wp{SILQ)M_R)G;1MMP%))T1 zNP7yM3M33p8E-k=%yNPZI-4 zNhYDnI8SJ$VENMd_2s7fG#)G%;#6Cvi{kmqC~)0WiCx}fRs8h?o{?#eV@=G|`_caM z4t&^j)cn{_S(*90iS0b5v0Jp%xISmAw*F0}sWzb4dRO9!zVPuTt>^pHUjI*xe3wbi zQlpJzX>aRe3v9)Ii!)T;C87mAdD`QPz&ID(UCnidyU#FK%$)6YYs##fn97R-^_Cf~ z1KJ62onfZrSEs^RfONVoutLiF@Wx&4DCCu<|6Fpi`(vz{=yDPq!hf|0T++JDr##*S zD-oDTsD!_<6x8@bD`G!nG26v#tjPtZXO+piDFK;($>Z{`fw33jD1%4!TCw*ay<}>6pJ+Id$7av)d z&;=E$uggUFXTRSfPRfw29=MVR>>EdTdJ-*E7`=R54VR=2(eK*AHA;WoF)FwHtdUFs zQc+hKiF6_QTVKFE-VlTl#=@X{$l`@7BY|A3QGjUDIZ;?e($B8APxznMX+W?{di1SS z|Fpf!R@ZB`j1NJ(SZ(9S&C97zm*Jw{z{*7Dr-XacCR2Fsvb|3&F2id)-hasEIa0tk zQj8%=1Kdv+r~4uTvY=a&HL6Emo>E4K+K*VYn$U;Vs_@dHQ}#KAhoKQseyla^32e|q z*3`dE_9j5QiRgWBh^bwsYs#djwP|Xpwb7>=#EkxGtU~k!H3!oLkFm(?QE!QDIWjas zb@UuILY>JR5#4;{M*TESC_oI9ona-c|Nbe<;=ue>TeHvv_1TIx^htudd>3d(`$S(T zXv_WlVy1Y?-uisw6&kMsd!SR^ivS@b%J3ai;a+eki5ZQf(&;Axh(R z_P>Gxi#6;i0HBmsl9Se54jXUJ;`TJwn)8BASdqqV`S%1;L_x)tAl7W6+2$aTl0!YrzH(FjDI@{W_P0mwbNg9WXd~> zK+zgS8^evv*U(aVC7;j4NrjpXKgkP|5%JB`&0|q`DapU?dyibckHwMkM&v{=_NIl= z!gqqJm5XwCUqay}=7w8BaNKF8?mZ2R$_LgIJuJl$F5<9$OmXb0q`{LED=~cTAbLT?;(>NaZ|E z5Y8Bx<P$;k1n(vzf~H!Cpmah;kxgu$wf6%l`iUx3D zzdg?bj3J>CsGz{(_v;hOvBYm&?;#=Sw|lA_xozSUAeJEY@wtgZ=w`FiSwH}O>E~1! zGHm?|KbDJ5sgi`1?3V3+HL#=(J@~vb6j> z8VRoWLoCteuy}K?o*t>`qztjNRN2Q zoKI&REEOD%#pa~^gu<7UU-IXbC9GZ#)(kpM13_sxO1{;4lEXybcH|_@>ey+Jvj9jm zA}!=XyWYJm7a_^UQ3z9(x`A3=<)nV=Cr{!^^NqxI4e5T0k^@ZuSI8U-^N#^gI;m)) zvL5Up`Zhg>n;of?4b5LfO@E!#ML%Wn@kXf(dZG-AlSEGo+lz0ZtNiG-%Na-t63Qcc zLWY7M}tu->M*H##&|~SOJIExmID1V1c|3kL)j!gCn$o^QX&YtshR& zc~|}TAAA>ECj|B5N~!a8W!;?0aB>Sg-46^P20&Y7CFY7)Ld~dk7B?CNa8if*AIE)r zD`E)T;b`F_0R|}4O8?iCC-P2 zl1qK$bn1Lz(0qerB>FMUqv;x|YG-Z=8p1cWA-o{rgCH%u{>fuet7LvvvD>=0@UNz| zJe<6bu(?ui`yasLchj}4H*1jd7XR*#DnzTapXl<6CUI=eJ+ zJvP#WG4@%^N|?Rq7s*%cZhT*0m_iyYDYhCOjMQE!L*loXID4Vqa@*tx|0FuUBm{Hv z`PECFN)MbDm_E~LjiMkH`llITXKV-SK406{)RD;r9WvGk#`cDDnf9iqLRQ;dC+9mk zW6+Z%t9Vb+j*GA-`lXaj{K*wU?$KrQMEdE7fJA6OA|2H`DAhYU;TqaaB!j6H3|zmP ziVP)m)=Z(!X&jL^ik!!{Acum{?BvS^Je1-90S1}y5a5I|3-->h5ZL|2-$YrN(OH6? zMSFKdap>Ar*eQA9KSBuB$|`axkwbH7{uks>w9%-yk7uhBmiB|VP#5GwcPiliAHZ!d4kpASF(BPg3nU3*D&D+uDA0ZI z6gEG%M}|^8g;b~D=E_aT^zIDh#)_QVr{3+WP6-fvfhKk!;^sTOA@cHk%|o{d7||HxtA z9e9X+JKe|ipE``PRXnc6Hw1Qa!y(QMf{U9Gav^52%*)E;IB^YdQ#u?5$97j#`wP8#f zWXa!cnTLUu7L@IpLoW&)h6_ap9u5}iGn)!c{X-{!<2m_F6Cz<7XZ-4FbQt&O7fpyA z9X>1!lN4Vvi>P5BxNkiA{X+|Bs;HROEjmsScHMP<0?!UU()8HgAisijzmkg%2IRAy zaPRM@vS~^s=hT{g!+~mrS-X?|IXecHAr~PRLECJP6^`iGYjHWN%r(!uiW(Q0Go@Jn z#pi(+6XW9JNFvFXaNnX*#B=&q;022jeL4h9=1?*9^sa@HEWbC@X55D0Qaz9LqgRC@Z%~$+7aQx0X|p$nqPAm*9B4STha@( z@qC1kR&QU?D=%~942}Q`x}AW$t8Lzl_P;ms3=AOql7~j?M>_J?|1LE6>4|imRrpL? zD|-v}cV^9+DM41)lFqlTq5*`YyYo12y<9N=$*sa=$U2C6Wi2~g)I&MIyrJC7G76r3 zXCj~!k|oGv7@idKxNCx=i+e#xdN6-p|D|Wb?Ymt8V>cQ^hp(-!l#;1F?=EChze4lCVUr zgqyT-d_yOBKrN+B8ib0zWSW81E7oc6{p}_No|15PDMWK?;0U-#;962t*dKs^zDhqo zLE)8URF-NL<_k?;u@*>H6($kSb^#y4T`D#XzGCoHY~|!1w7dOFB9{PXd#YwC+*2S; zI>2NPvAtXS_@qhR>fVGmS(BJdMZ**b6irst7X$aLTeIAk4PUH~;<$@Pv>GAvRLwY9 zvN3s+24(3qVQF6ybIm`p7^pBl7URfDRva6UMWa!rH1|>`jvT{PT;o2b0&u{%+DI-O zieZUanZHdH7Ep;dC(@@{4-)tecYfR2hD>&Lil>8Kg7z zFMy7bPmZF=Apylf-}T*QJHL(2$8aXBGxr$eM4ld$-zr>aD*Z*uzX`F?B{^#>MJ!%G zm;iOqY37gTFpW*7AAt3B?4(y_Y3EBplH05xz4ND^5zpzgM^@aQ<5KS|G3txF{4{Gnj4+Q$CBboK@hUayi51F z`T;-q-l(%F`dBK@AUW(%vYo~+xQBv0DO>#({MTDfkX2mcpZ8Kr46{Ybz3bn|%YNbw zffF?PQ`=DCYz#-3 zh0AAwbeYx2FB_YB+seA#AqJ+daVQ~(li6RZF}9Y*{qCK%G?sT<6I6k_sqZ9+SJc zxZu4|&!~KKnmwg^3%;B2NwpEDau}uDIWkkO+CrEiCh|6u#X`Dw)hd95erk$cW<} z1jiZuLIN7fd)EPUIX&PZ4~88C9tH(_8dXwCvK?`V8W)dQE5zE8J%#0b6^5Fv1A;A$d^j2p(a*TI+BM+-(h=HXFI+Q%-T9 zn-I*q06Y(nDj4$k$3oI|XCP|~zjCEWnp8hh>S;A;xKe-gIx@z-?unSt?H$7{qZW54 zYrsyl+nV*g`4U}&D4$a#`0v|(g`E*pyM+cUR@x_Fd}{^@5fS7VNe%j5O~4S^`g8Ir z;x5g?m1*3ALeu?u2u+He$Ug!Bc!K5?8o8!?k~ci;HrovIx0WsSnW1W==O7-vW|fWDpvs z6?#pp`-|9r_7%T~srYw;oxW!@#5xg-26KGt5%g|GjmQMqokXyBPpB{_pf$$${;B+1 z39VPTb4_Jm&F|eOelk$5YgFB1^1FU{UP;N&e(1mZX9-{P7t?-T6EhEy5fnbN+x~NL z{Ce$76`iawLJ4FWWSyo83I<)Y34NDp9ub0twA>DQuhdnnLWHVQvS%v06MNWep<^qr zUA4wuX(8^g;#+IuAw<&L=(N4vJ=?_Qyyn@>Hs|rHY$jqzY9C%M({@^4G22@G~@TK`q9MmS-AOML|s^d&=@y^Fu&->*K95lC+Udw zw~~q)VPNCxI$8W}%193@l4^HuKi6DaprRw{-VY+VggyBcQ9Mi8)N89i%~ov!HN=?J zd4wwLiYWVFuBfE)+2)yh=fR||K$@|^z1c`eZ)c4W)dE9lNS!*;GB^0d9A)U}mwkbk zJR5f;54%jc9H8d&gZn!Z| zs69gnb*omEUiJWA*>{#|8^-r*oa@G<)e}EBq$(WXhS=JVJ2Dg*#>jBMOEk8QJ4I0b z8o|WGO?d%7eOflrvbXft!9-jbw&7*@U+!yr_n#xZ6=|#`cZ@!#j>VjM??695*Mma z_QSKp{=)oKrG5dO2QCAcAlEJHNcQ!F4_+P_wZo1x=5$l(<7`^x5A_>ch~B-9lS;Zg zyUu((Z&J9NZn@St9q3TUc&f5mZv}9rHw67|bML&IB%Qc_#zC_y&UoZI(@dfy7yG?U zN+ZMrZuq2}N>r#+E}?xQYo$pG}ZT~q|r zqbB~#q-WtwspSG|QHPAq`5YHwD-xWpyPoLWim%y#i(wgbI+^0mm zJmRm;ftvVAe9~W)FD(V2Awi}E2Z=3KKMUE7Kw}2*pJwS|R&C5%jYrw;cZNJp%L+C& zKDL#w68h8d1*@m(VgA=WDdagDij{);d3I`SMONz~H?#Ngc;x*FJ`?_jo*Wyr1E$Xh zUeh|$9aDxc-+sB3F7R;Uc>YjH@e6vNH<^9*f$1l$1Iq1}?R7c~s(YT`EbVnY5U?xe zQ++NY1M8VSvWw;K?kNS?0`@QXxESwB7P-X-lfqP@2elHrxv5bSK^Z)(oibd@WQn*4!XhDpw__t z;!-$jA1;zRdBhGAra=5->ViD|nlq=)7;{~4%9mlbYgOU0@~YkZ3SwHI zC(n#9;xk}wI++@_+t8n8;ya8^|cZum+8oA!HiD^?q#9d>TRGIB7KD9x@a!~2OQL946_KhYuA7fQ@JJI+!YKx$vM zhLM08ij=t^@5t+c-@cmuhzKdb?m}q<=Nd(W&Wv*k^9xpbzTX#85`DzOdOQYx9d|4T zdgz`36-qKJ9X2b&_$|W2EWYIhz{MCn9n63yo>hX&Dt+@DpR!pW_T5A$xQ0vd{-7hb z=;g7a^3zg4TO@sI(cU{}za&FIJ0TZ|#*c;uVq<~?L7hYjTQH$XPDSzD>}cv7{^7Qn z8_MAVy#oymv_#RotV^grzn+YBj5>^FlGvp#+2^`q!eJandJpnFZQ=uYA!<`e)4|es zq3wRC!QGbd2lAtdYm<3keED`&ox#P}Qd7f^?GlfIexhg&jPm}Vq-gLk$=*RM z$^`;`Mx-IXu6VJjuZ)FSB#@diphWyy9={0D?4J(tj@Wu*kKMix4i3oCcbE6Y?M8ey zd^6uU!(N-V>L6i(wN&q$REC<5;XjnRtG!4S+r+Osva#2|0&&=zj2xR6@5mMbQow0o z!b*;S+R2csojOaKNce#-|e(7pUnceRhN#F8&G!Q0>Xl{T`v_qvbv9L-=%HHp(n%scdrIxe}Xm3>kHN#<6 zxzNU&=k8CA8=Bi5%Kg>LiYSB_Li=ACe_(rMY5wSnl&<@U)s`5x&e@PpIVn+uZ1q1; zcY$()K0$*%frKQ(*EWTsH&E_36r69ozWKU9Osq|t9P*im=3qPn|AA+V%$Sq&q>D)Ierh&k-mF1aQ}{g zLHR?yp>qA9hk9>rWQMxz%zV$goD)@-b$uyeNq@&c(W6-8Hj6JT@wfFR$7yK7JzQ49 zYDWM3wgAqA5X$j?FU{;IvG{_R_jeabh!hWp6V3Tghalaa_M3g7ni}NN&^k~h4{$8o zxmSt%BKA{sq(DindkrOyW1B4cgy5Gb&PAiXNDr%kTf^#ZAD!Lsu%8km1vE^N#F0_6 z5f0&J@iUDkPMzfule5ythECVrhBM9j8loJuKy(oGNaoOmBEy=#v9*2tCTPUnT2>ygYM5d~%Kq8v78M`+ zkV9R|t_bHPLtUH;i9N{)=-5kQ8>dCyLp9jm*Ioo8#t42ws)afAoosDkZ+wq1Ph&NXCnIcudENe3c$06 + + + + + + + + + + + + diff --git a/src/Oolong.jl b/src/Oolong.jl index 8b1c36b..8c30948 100644 --- a/src/Oolong.jl +++ b/src/Oolong.jl @@ -3,10 +3,7 @@ module Oolong const OL = Oolong export OL +include("config.jl") include("core.jl") -function __init__() - start() -end - end diff --git a/src/config.jl b/src/config.jl new file mode 100644 index 0000000..d583ce4 --- /dev/null +++ b/src/config.jl @@ -0,0 +1,7 @@ +using Configurations +using YAML + +@option struct Config + banner::Bool = Base.JLOptions().banner != 0 + color::Bool = Base.have_color +end \ No newline at end of file diff --git a/src/core.jl b/src/core.jl index 58d7ddd..3656253 100644 --- a/src/core.jl +++ b/src/core.jl @@ -9,6 +9,91 @@ using Dates const KEY = :OOLONG +##### +# basic +##### + +""" +Similar to `Future`, but it will unwrap inner `Future` or `Promise` recursively when trying to get the *promised* value. +""" +struct Promise + f::Future + function Promise(args...) + new(Future(args...)) + end +end + +function Base.getindex(p::Promise) + x = p.f[] + while x isa Promise || x isa Future + x = x[] + end + x +end + +Base.put!(p::Promise, x) = put!(p.f, x) + +##### + +struct PotNotRegisteredError <: Exception + pid::PotID +end + +Base.showerror(io::IO, err::PotNotRegisteredError) = print(io, "can not find any pot associated with the pid: $(err.pid)") + +##### + +struct RequireInfo + cpu::Float64 + gpu::Float64 +end + +Base.:(<=)(x::RequireInfo, y::RequireInfo) = x.cpu <= y.cpu && x.gpu <= y.gpu +Base.:(-)(x::RequireInfo, y::RequireInfo) = RequireInfo(x.cpu - y.cpu, x.gpu - y.gpu) + +struct RequirementNotSatisfiedError <: Exception + required::RequireInfo + remaining::RequireInfo +end + +Base.showerror(io::IO, err::RequirementNotSatisfiedError) = print(io, "required: $(err.required), remaining: $(err.remaining)") + +##### + +"System level messages are processed immediately" +abstract type AbstractSysMsg end + +is_prioritized(msg) = false +is_prioritized(msg::AbstractSysMsg) = true + +# !!! force system level messages to be executed immediately +# directly copied from +# https://github.com/JuliaLang/julia/blob/6aaedecc447e3d8226d5027fb13d0c3cbfbfea2a/base/channels.jl#L13-L31 +# with minor modification +function Base.put_buffered(c::Channel, v) + lock(c) + try + while length(c.data) == c.sz_max + Base.check_channel_state(c) + wait(c.cond_put) + end + if is_prioritized(v) + pushfirst!(c.data, v) # !!! force sys msg to be handled immediately + else + push!(c.data, v) # !!! force sys msg to be handled immediately + end + # notify all, since some of the waiters may be on a "fetch" call. + notify(c.cond_take, nothing, true, false) + finally + unlock(c) + end + return v +end + +##### +# PotID +##### + struct PotID path::Tuple{Vararg{Symbol}} end @@ -27,7 +112,7 @@ end function Base.show(io::IO, p::PotID) if isempty(getfield(p, :path)) - print(io, '/') + print(io, "/") else for x in getfield(p, :path) print(io, '/') @@ -41,54 +126,132 @@ function PotID(s::String) if s[1] == '/' PotID(Tuple(Symbol(x) for x in split(s, '/';keepempty=false))) else - self() * s + self() / s end else PotID(()) end end -Base.:(*)(p::PotID, s::String) = PotID((getfield(p, :path)..., (Symbol(x) for x in split(s, '/';keepempty=false))...)) +Base.:(/)(p::PotID, s::String) = PotID((getfield(p, :path)..., (Symbol(x) for x in split(s, '/';keepempty=false))...)) const ROOT = P"/" const LOGGER = P"/log" const SCHEDULER = P"/scheduler" const USER = P"/user" +##### +# Message Processing +##### -# struct Success{V} -# value::V -# end +process(tea, args...;kw...) = tea(args...;kw...) -# Base.getindex(s::Success{<:Success}) = s.value[] -# Base.getindex(s::Success) = s.value +##### -# struct Failure{E} -# error::E -# end +function Base.put!(p::PotID, flavor) + try + put!(p[], flavor) + catch e + # TODO add test + if e isa PotNotRegisteredError + rethrow(e) + else + @error e + boil(p) + put!(p, flavor) + end + end +end + +##### -# Failure() = Failure(nothing) +struct CallMsg{A} + args::A + kw + promise +end -# Base.getindex(f::Failure{<:Failure}) = f.error[] -# Base.getindex(f::Failure) = f.error +is_prioritized(::CallMsg{<:Tuple{<:AbstractSysMsg}}) = true -"Similar to `Future`, but it will unwrap inner `Future` or `Promise` recursively." -struct Promise - f::Future - function Promise(args...) - new(Future(args...)) +function (p::PotID)(args...;kw...) + promise = Promise(whereis(p)) # !!! the result should reside in the same place + put!(p, CallMsg(args, kw.data, promise)) + promise +end + +# ??? non specialized tea? +function process(tea, msg::CallMsg) + try + res = process(tea, msg.args...;msg.kw...) + put!(msg.promise, res) + catch err + # avoid dead lock + put!(msg.promise, err) end end -function Base.getindex(p::Promise) - x = p.f[] - while x isa Promise || x isa Future - x = x[] +##### + +function Base.:(|>)(x, p::PotID) + put!(p, x) + nothing +end + +##### + +struct GetPropMsg + prop::Symbol +end + +Base.getproperty(p::PotID, prop::Symbol) = p(GetPropMsg(prop)) + +process(tea, msg::GetPropMsg) = getproperty(tea, msg.prop) + +##### SysMsg + +""" +Signal a Pot to close the channel and release claimed resources. +By default, all children are closed recursively. +""" +struct CloseMsg <: AbstractSysMsg +end + +Base.close(p::PotID) = p(CloseMsg()) + +function process(tea, ::CloseMsg) + for c in children() + c(CloseMsg())[] end - x end -Base.put!(p::Promise, x) = put!(p.f, x) +""" +Close the active channel and remove the registered `Pot`. +""" +struct RemoveMsg <: AbstractSysMsg +end + +Base.rm(p::PotID) = p(RemoveMsg()) + +function process(tea, ::RemoveMsg) + for c in children() + c(RemoveMsg())[] + end + unregister(self()) +end + +struct ResumeMsg <: AbstractSysMsg +end + +process(tea, ::ResumeMsg) = nothing + +struct RestartMsg <: AbstractSysMsg +end + +struct PreRestartMsg <: AbstractSysMsg +end + +struct PostRestartMsg <: AbstractSysMsg +end ##### # Logging @@ -121,8 +284,8 @@ function (L::DefaultLogger)(msg::LogMsg) level, _module, group, id, file, line ) + printstyled(iob, "$(kw.datetime) "; color=:light_black) printstyled(iob, prefix; bold=true, color=color) - printstyled(iob, "$(kw.datetime)"; color=:light_black) printstyled(iob, "[$(kw.from)@$(kw.myid)]"; color=:green) print(iob, message) for (k,v) in pairs(kw) @@ -154,14 +317,6 @@ end # Pot Definition ##### -struct RequireInfo - cpu::Float64 - gpu::Float64 -end - -Base.:(<=)(x::RequireInfo, y::RequireInfo) = x.cpu <= y.cpu && x.gpu <= y.gpu -Base.:(-)(x::RequireInfo, y::RequireInfo) = RequireInfo(x.cpu - y.cpu, x.gpu - y.gpu) - struct Pot tea_bag::Any pid::PotID @@ -199,27 +354,12 @@ end _self() = get!(task_local_storage(), KEY, PotState(USER, current_task())) self() = _self().pid -local_scheduler() = SCHEDULER*"local_scheduler_$(myid())" +local_scheduler() = SCHEDULER/"local_scheduler_$(myid())" Base.parent() = parent(self()) Base.parent(p::PotID) = PotID(getfield(p, :path[1:end-1])) -##### -# Exceptions -##### - -struct PotNotRegisteredError <: Exception - pid::PotID -end - -Base.showerror(io::IO, err::PotNotRegisteredError) = print(io, "can not find any pot associated with the pid: $(err.pid)") - -struct RequirementNotSatisfiedError <: Exception - required::RequireInfo - remaining::RequireInfo -end - -Base.showerror(io::IO, err::RequirementNotSatisfiedError) = print(io, "required: $(err.required), remaining: $(err.remaining)") +children() = children(self()) ##### # Pot Scheduling @@ -227,18 +367,46 @@ Base.showerror(io::IO, err::RequirementNotSatisfiedError) = print(io, "required: # TODO: set ttl? -"local cache to reduce remote call, we may use redis like db later" +""" +Local cache on each worker to reduce remote call. +The links may be staled. +""" const POT_LINK_CACHE = Dict{PotID, RemoteChannel{Channel{Any}}}() -const POT_REGISTRY_CACHE = Dict{PotID, Pot}() + +""" +Only valid on the driver to keep track of all registered pots. +TODO: use a kv db +""" +const POT_REGISTRY = Dict{PotID, Pot}() +const POT_CHILDREN = Dict{PotID, Set{PotID}}() + +function is_registered(p::Pot) + is_exist = remotecall_wait(1) do + haskey(Oolong.POT_REGISTRY, p.pid) + end + is_exist[] +end function register(p::Pot) - POT_REGISTRY_CACHE[p.pid] = p - if myid() != 1 - remotecall_wait(1) do - Oolong.POT_REGISTRY_CACHE[p.pid] = p - end + remotecall_wait(1) do + Oolong.POT_REGISTRY[p.pid] = p + children = get!(Oolong.POT_CHILDREN, parent(p.pid), Set{PotID}()) + push!(children, p.pid) + end +end + +function unregister(p::PotID) + remotecall_wait(1) do + delete!(Oolong.POT_REGISTRY, p) + delete!(Oolong.POT_CHILDREN, p) + end +end + +function children(p::PotID) + remotecall_wait(1) do + # ??? data race + get!(Oolong.POT_CHILDREN, p, Set{PotID}()) end - p end function link(p::PotID, ch::RemoteChannel) @@ -266,18 +434,20 @@ end whereis(p::PotID) = p[].where function Base.getindex(p::PotID, ::typeof(!)) - get!(POT_REGISTRY_CACHE, p) do - pot = remotecall_wait(1) do - get(Oolong.POT_REGISTRY_CACHE, p, nothing) - end - if isnothing(pot[]) - throw(PotNotRegisteredError(p)) - else - pot[] - end + pot = remotecall_wait(1) do + get(Oolong.POT_REGISTRY, p, nothing) + end + if isnothing(pot[]) + throw(PotNotRegisteredError(p)) + else + pot[] end end +""" +For debug only. Only a snapshot is returned. +!!! DO NOT MODIFY THE RESULT DIRECTLY +""" Base.getindex(p::PotID, ::typeof(*)) = p(_self())[] local_boil(p::PotID) = local_boil(p[!]) @@ -293,8 +463,25 @@ function local_boil(p::Pot) try flavor = take!(ch) process(tea, flavor) + if flavor isa CloseMsg || flavor isa RemoveMsg + break + end catch err - @error err + @debug err + flavor = parent()(err)[] + if msg isa ResumeMsg + process(tea, flavor) + elseif msg isa CloseMsg + process(tea, flavor) + break + elseif msg isa RestartMsg + process(tea, PreRestartMsg()) + tea = tea_bag() + process(tea, PostRestartMsg()) + else + @error "unknown msg received from parent: $exec" + rethrow() + end finally end end @@ -449,67 +636,6 @@ function (s::Scheduler)(p::PotID) res end -##### -# Message passing -##### - -function Base.put!(p::PotID, flavor) - try - put!(p[], flavor) - catch e - # TODO add test - if e isa PotNotRegisteredError - rethrow(e) - else - @error e - boil(p) - put!(p, flavor) - end - end -end - -process(tea, flavor) = tea(flavor) - -# CallMsg - -struct CallMsg{A} - args::A - kw - promise -end - -function (p::PotID)(args...;kw...) - promise = Promise(whereis(p)) # !!! the result should reside in the same place - put!(p, CallMsg(args, kw.data, promise)) - promise -end - -# ??? non specialized tea? -function process(tea, msg::CallMsg) - try - res = handle(tea, msg.args...;msg.kw...) - put!(msg.promise, res) - catch err - # avoid dead lock - put!(msg.promise, err) - end -end - -handle(tea, args...;kw...) = tea(args...;kw...) - -# CastMsg - -Base.:(|>)(x, p::PotID) = put!(p, x) - -# GetPropMsg - -struct GetPropMsg - prop::Symbol -end - -Base.getproperty(p::PotID, prop::Symbol) = p(GetPropMsg(prop)) - -handle(tea, msg::GetPropMsg) = getproperty(tea, msg.prop) ##### # System Initialization @@ -525,7 +651,56 @@ struct Root end end -function start() +function banner(io::IO=stdout;color=true) + c = Base.text_colors + tx = c[:normal] # text + d1 = c[:bold] * c[:blue] # first dot + d2 = c[:bold] * c[:red] # second dot + d3 = c[:bold] * c[:green] # third dot + d4 = c[:bold] * c[:magenta] # fourth dot + + if color + print(io, + """ + ____ _ | > 是非成败转头空 + / $(d1)__$(tx) \\ | | | > Success or failure, + | $(d1)| |$(tx) | ___ | | ___ _ __ __ _ | > right or wrong, + | $(d1)| |$(tx) |/ $(d2)_$(tx) \\| |/ $(d3)_$(tx) \\| '_ \\ / $(d4)_$(tx)` | | > all turn out vain. + | $(d1)|__|$(tx) | $(d2)(_)$(tx) | | $(d3)(_)$(tx) | | | | $(d4)(_)$(tx) | | + \\____/ \\___/|_|\\___/|_| |_|\\__, | | The Immortals by the River + __/ | | -- Yang Shen + |___/ | (Translated by Xu Yuanchong) + """) + else + print(io, + """ + ____ _ | > 是非成败转头空 + / __ \\ | | | > Success or failure, + | | | | ___ | | ___ _ __ __ _ | > right or wrong, + | | | |/ _ \\| |/ _ \\| '_ \\ / _` | | > all turn out vain. + | |__| | (_) | | (_) | | | | (_) | | + \\____/ \\___/|_|\\___/|_| |_|\\__, | | The Immortals by the River + __/ | | -- Yang Shen + |___/ | (Translated by Xu Yuanchong) + """) + end +end + +function start(config_file::String="Oolong.yaml";kw...) + config = nothing + if isfile(config_file) + @info "Found $config_file. Loading configs..." + config = Configurations.from_dict(Config, YAML.load_file(config_file; dicttype=Dict{String, Any});kw...) + else + @info "$config_file not found. Using default configs." + config = Config(;kw...) + end + start(config) +end + +function start(config::Config) + config.banner && banner(color=config.color) + @info "$(@__MODULE__) starting..." if myid() == 1 local_boil(@pot Root() name=ROOT logger=current_logger()) @@ -535,3 +710,6 @@ function start() local_boil(@pot LocalScheduler() name=local_scheduler()) end end + +function stop() +end diff --git a/test/runtests.jl b/test/runtests.jl index 961380e..2959e90 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -4,5 +4,4 @@ using Base.Threads @testset "Oolong.jl" begin include("core.jl") - include("serve.jl") end diff --git a/test/serve.jl b/test/serve.jl deleted file mode 100644 index 60ee35c..0000000 --- a/test/serve.jl +++ /dev/null @@ -1,52 +0,0 @@ -@testset "serve" begin - # q: request - # p: reply - # |: batch_wait_timeout_s reached - - @testset "batch_wait_timeout_s = 0" begin - # case 1: qpqpqp - struct DummyModel end - - function OL.handle(m::DummyModel, msg::OL.RequestMsg) - xs = msg.msg - @debug "value received in model" xs typeof(xs) - sleep(0.1) - res = vec(sum(xs; dims=1)) .+ 1 - OL.async_rep(msg.from, res) - end - model = @actor DummyModel() name="model" - - server = @actor OL.BatchStrategy(model;max_batch_size=2) name="server" - - worker = OL.Mailbox() - t = @elapsed for i in 1:10 - put!(server,OL.RequestMsg([i], worker)) - sleep(0.11) - end - - for i in 1:10 - msg = take!(worker) - @test msg.msg == i+1 - end - - # case 2: qqqqqpqqqqp - t = @elapsed begin - for i in 1:5 - put!(server,OL.RequestMsg([i], worker)) - end - for i in 1:5 - @test take!(worker).msg == i+1 - end - end - - t = @elapsed begin - for i in 1:64 - put!(server,OL.RequestMsg([i], worker)) - end - for i in 1:64 - @test take!(worker).msg == i+1 - end - end - end - -end \ No newline at end of file From 4497116abca0aa063ee48f5b8bd13c2d07ec7e0b Mon Sep 17 00:00:00 2001 From: Jun Tian Date: Tue, 14 Sep 2021 00:50:15 +0800 Subject: [PATCH 4/4] sync local changes --- .gitignore | 1 + Oolong.toml | 13 + Project.toml | 5 +- README.md | 6 +- src/Oolong.jl | 5 +- src/base.jl | 158 ++++++++ src/config.jl | 31 ++ src/core.jl | 715 ----------------------------------- src/core/core.jl | 4 + src/core/message_handling.jl | 136 +++++++ src/core/pot.jl | 47 +++ src/core/pot_id.jl | 49 +++ src/core/scheduling.jl | 272 +++++++++++++ src/logging.jl | 148 ++++++++ src/start.jl | 70 ++++ 15 files changed, 941 insertions(+), 719 deletions(-) create mode 100644 Oolong.toml create mode 100644 src/base.jl delete mode 100644 src/core.jl create mode 100644 src/core/core.jl create mode 100644 src/core/message_handling.jl create mode 100644 src/core/pot.jl create mode 100644 src/core/pot_id.jl create mode 100644 src/core/scheduling.jl create mode 100644 src/logging.jl create mode 100644 src/start.jl diff --git a/.gitignore b/.gitignore index b067edd..976666b 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /Manifest.toml +/logs diff --git a/Oolong.toml b/Oolong.toml new file mode 100644 index 0000000..6b577dc --- /dev/null +++ b/Oolong.toml @@ -0,0 +1,13 @@ +banner = true +color = true + +[logging] +log_level = "Debug" +date_format = "yyyy-mm-ddTHH:MM:SS.s" + + [logging.driver_logger] + console_logger.is_expand_stack_trace = true + rotating_logger.path = "./logs" + rotating_logger.file_format = "YYYY-mm-dd.\\l\\o\\g" + + [logging.loki_logger] \ No newline at end of file diff --git a/Project.toml b/Project.toml index 29fd500..a17f3b2 100644 --- a/Project.toml +++ b/Project.toml @@ -1,14 +1,17 @@ name = "Oolong" uuid = "c9dcc2fc-6356-41de-aa29-480ea90c21cd" authors = ["Jun Tian and contributors"] -version = "0.1.0" +version = "0.0.1" [deps] CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" Configurations = "5218b696-f38b-4ac9-8b61-a12ec717816d" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" +GarishPrint = "b0ab02a7-8576-43f7-aa76-eaa7c3897c54" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" +LoggingExtras = "e6f89c97-d47a-5376-807f-9c37f3926c36" +LokiLogger = "51d429d1-9683-4c89-86d7-889f440454ef" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" YAML = "ddb6d928-2868-570f-bddf-ab3f9cf99eb6" diff --git a/README.md b/README.md index 07bb244..e23dbe8 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ Please contact us if you have a concrete scenario but not sure how to use this p 3. ☑️ CPU/GPU allocation 4. 🧐 Auto intall+using dependencies 5. ☑️ Global configuration + 6. 🧐 Close pot when it is idle for a period 3. Example usages 1. 🧐 Parameter search 2. 🧐 Batch evaluation. @@ -80,9 +81,10 @@ Please contact us if you have a concrete scenario but not sure how to use this p 2. Dashboard 1. 🧐 [grafana](https://grafana.com/) 3. Custom Logger - 1. 🧐 [LokiLogger.jl](https://github.com/fredrikekre/LokiLogger.jl) + 1. ☑️ [LokiLogger.jl](https://github.com/fredrikekre/LokiLogger.jl) 2. 🧐 [Stipple.jl](https://github.com/GenieFramework/Stipple.jl) 4. Tracing + 1. [opentelemetry](https://opentelemetry.io/) 1. Stage 3 1. Drop out Distributed.jl? 1. 🧐 `Future` will transfer the ownership of the underlying data to the caller. Not very efficient when the data is passed back and forth several times in its life circle. @@ -133,7 +135,7 @@ A `Pot` is mainly a container of an arbitrary object (`tea`) which is instantiat The following design decisions need to be reviewed continuously. -1. Each `Pot` can only be created inside of another `Pot`, which forms a child-parent relation. If no `Pot` is found in the `current_task()`, the parent is bind to `/user` by default. +1. Each `Pot` can only be created inside of another `Pot`, which forms a child-parent relation. If no `Pot` is found in the `current_task()`, the parent is bind to `/user` by default. When registering a new `Pot` whose`PotID` is already registerred. The old one will be removed first. This will allow updating `Pot`s dynamically. (Do we really need this feature?) ### FAQ diff --git a/src/Oolong.jl b/src/Oolong.jl index 8c30948..346147e 100644 --- a/src/Oolong.jl +++ b/src/Oolong.jl @@ -4,6 +4,9 @@ const OL = Oolong export OL include("config.jl") -include("core.jl") +include("logging.jl") +include("base.jl") +include("core/core.jl") +include("start.jl") end diff --git a/src/base.jl b/src/base.jl new file mode 100644 index 0000000..bf3c201 --- /dev/null +++ b/src/base.jl @@ -0,0 +1,158 @@ +const KEY = :OOLONG + +""" +Similar to `Future`, but we added some customized methods. +""" +struct Promise + f::Future + function Promise(args...) + new(Future(args...)) + end +end + +Base.getindex(p::Promise) = getindex(p.f) +Base.wait(p::Promise) = wait(p.f) + +"Recursively fetch inner value" +function Base.getindex(p::Promise, ::typeof(!)) + x = p.f[] + while x isa Promise || x isa Future + x = x[] + end + x +end + +function Base.getindex(ps::Vector{Promise}) + res = Vector(undef, length(ps)) + @sync for (i, p) in enumerate(ps) + Threads.@spawn begin + res[i] = p[] + end + end + res +end + +struct TimeOutError{T} <: Exception + t::T +end + +Base.showerror(io::IO, err::TimeOutError) = print(io, "failed to complete in $(err.t) seconds") + +""" + p::Promise[t::Number] + +Try to fetch value during a period of `t`. +A [`TimeOutError`](@ref) is thrown if the underlying data is still not ready after `t`. +""" +function Base.getindex(p::Promise, t::Number, pollint=0.1) + res = timedwait(t;pollint=pollint) do + isready(p) + end + if res === :ok + p[] + else + throw(TimeOutError(t)) + end +end + +Base.put!(p::Promise, x) = put!(p.f, x) +Base.isready(p::Promise) = isready(p.f) + +##### + +struct PotNotRegisteredError <: Exception + pid::PotID +end + +Base.showerror(io::IO, err::PotNotRegisteredError) = print(io, "can not find any pot associated with the pid: $(err.pid)") + +##### + +const RESOURCE_REGISTRY = Dict{Symbol, UInt}( + :cpu => () -> Threads.nthreads(), + :gpu => () -> length(CUDA.devices()) +) + +struct ResourceInfo{I<:NamedTuple} + info::I +end + +ResourceInfo() = ResourceInfo(NamedTuple(k=>v() for (k,v) in RESOURCE_REGISTRY)) + +ResourceInfo(;kw...) = ResourceInfo(kw.data) +Base.keys(r::ResourceInfo) = keys(r.info) +Base.getindex(r::ResourceInfo, x) = getindex(r.info, x) +Base.haskey(r::ResourceInfo, x) = haskey(r.info, x) + +function Base.:(<=)(x::ResourceInfo, y::ResourceInfo) + le = true + for k in keys(x) + if haskey(y, k) && x[k] <= y[k] + continue + else + le = false + break + end + end + le +end + +function Base.:(-)(x::ResourceInfo, y::ResourceInfo) + merge(x, (k => x[k]-v for (k,v) in pairs(y))) +end + +struct RequirementNotSatisfiedError <: Exception + required::ResourceInfo + remaining::ResourceInfo +end + +Base.showerror(io::IO, err::RequirementNotSatisfiedError) = print(io, "required: $(err.required), remaining: $(err.remaining)") + +##### + +"System level messages are processed immediately" +abstract type AbstractSysMsg end + +is_prioritized(msg) = false +is_prioritized(msg::AbstractSysMsg) = true + +# !!! force system level messages to be executed immediately +# directly copied from +# https://github.com/JuliaLang/julia/blob/6aaedecc447e3d8226d5027fb13d0c3cbfbfea2a/base/channels.jl#L13-L31 +# with minor modification +function Base.put_buffered(c::Channel, v) + lock(c) + try + while length(c.data) == c.sz_max + Base.check_channel_state(c) + wait(c.cond_put) + end + if is_prioritized(v) + pushfirst!(c.data, v) # !!! force sys msg to be handled immediately + else + push!(c.data, v) # !!! force sys msg to be handled immediately + end + # notify all, since some of the waiters may be on a "fetch" call. + notify(c.cond_take, nothing, true, false) + finally + unlock(c) + end + return v +end + +##### + +"Similar to `RemoteException`, except that we need the `PotID` info." +struct Failure <: Exception + pid::PotID + captured::CapturedException +end + +Failure(captured) = Failure(self(), captured) + +is_prioritized(::Failure) = true + +function Base.showerror(io::IO, f::Failure) + println(io, "In pot $(f.pid) :") + showerror(io, re.captured) +end diff --git a/src/config.jl b/src/config.jl index d583ce4..63754c3 100644 --- a/src/config.jl +++ b/src/config.jl @@ -1,7 +1,38 @@ using Configurations using YAML +using Logging +using Dates + +@option struct ConsoleLoggerConfig + is_expand_stack_trace::Bool = true +end + +@option struct RotatingLoggerConfig + path::String = "logs" + file_format::String = raw"YYYY-mm-dd.\l\o\g" +end + +@option struct DriverLoggerConfig + console_logger::Union{ConsoleLoggerConfig, Nothing}=ConsoleLoggerConfig() + rotating_logger::Union{RotatingLoggerConfig, Nothing}=RotatingLoggerConfig() +end + +@option struct LokiLoggerConfig + url::String = "http://127.0.0.1:3100" +end + +@option struct LoggingConfig + # filter + log_level::String = "Info" + # transformer + date_format::String="yyyy-mm-ddTHH:MM:SS.s" + # sink + driver_logger::Union{DriverLoggerConfig, Nothing} = DriverLoggerConfig() + loki_logger::Union{LokiLoggerConfig, Nothing} = nothing +end @option struct Config banner::Bool = Base.JLOptions().banner != 0 color::Bool = Base.have_color + logging::LoggingConfig = LoggingConfig() end \ No newline at end of file diff --git a/src/core.jl b/src/core.jl deleted file mode 100644 index 3656253..0000000 --- a/src/core.jl +++ /dev/null @@ -1,715 +0,0 @@ -export @P_str, @pot - -using Distributed -using UUIDs:uuid4 -using Base.Threads -using CUDA -using Logging -using Dates - -const KEY = :OOLONG - -##### -# basic -##### - -""" -Similar to `Future`, but it will unwrap inner `Future` or `Promise` recursively when trying to get the *promised* value. -""" -struct Promise - f::Future - function Promise(args...) - new(Future(args...)) - end -end - -function Base.getindex(p::Promise) - x = p.f[] - while x isa Promise || x isa Future - x = x[] - end - x -end - -Base.put!(p::Promise, x) = put!(p.f, x) - -##### - -struct PotNotRegisteredError <: Exception - pid::PotID -end - -Base.showerror(io::IO, err::PotNotRegisteredError) = print(io, "can not find any pot associated with the pid: $(err.pid)") - -##### - -struct RequireInfo - cpu::Float64 - gpu::Float64 -end - -Base.:(<=)(x::RequireInfo, y::RequireInfo) = x.cpu <= y.cpu && x.gpu <= y.gpu -Base.:(-)(x::RequireInfo, y::RequireInfo) = RequireInfo(x.cpu - y.cpu, x.gpu - y.gpu) - -struct RequirementNotSatisfiedError <: Exception - required::RequireInfo - remaining::RequireInfo -end - -Base.showerror(io::IO, err::RequirementNotSatisfiedError) = print(io, "required: $(err.required), remaining: $(err.remaining)") - -##### - -"System level messages are processed immediately" -abstract type AbstractSysMsg end - -is_prioritized(msg) = false -is_prioritized(msg::AbstractSysMsg) = true - -# !!! force system level messages to be executed immediately -# directly copied from -# https://github.com/JuliaLang/julia/blob/6aaedecc447e3d8226d5027fb13d0c3cbfbfea2a/base/channels.jl#L13-L31 -# with minor modification -function Base.put_buffered(c::Channel, v) - lock(c) - try - while length(c.data) == c.sz_max - Base.check_channel_state(c) - wait(c.cond_put) - end - if is_prioritized(v) - pushfirst!(c.data, v) # !!! force sys msg to be handled immediately - else - push!(c.data, v) # !!! force sys msg to be handled immediately - end - # notify all, since some of the waiters may be on a "fetch" call. - notify(c.cond_take, nothing, true, false) - finally - unlock(c) - end - return v -end - -##### -# PotID -##### - -struct PotID - path::Tuple{Vararg{Symbol}} -end - -""" - P"[/]your/pot/path" - -The path can be either relative or absolute path. If a relative path is provided, it will be resolved to an absolute path based on the current context. - -!!! note - We don't validate the path for you during construction. A [`PotNotRegisteredError`](@ref) will be thrown when you try to send messages to an unregistered path. -""" -macro P_str(s) - PotID(s) -end - -function Base.show(io::IO, p::PotID) - if isempty(getfield(p, :path)) - print(io, "/") - else - for x in getfield(p, :path) - print(io, '/') - print(io, x) - end - end -end - -function PotID(s::String) - if length(s) > 0 - if s[1] == '/' - PotID(Tuple(Symbol(x) for x in split(s, '/';keepempty=false))) - else - self() / s - end - else - PotID(()) - end -end - -Base.:(/)(p::PotID, s::String) = PotID((getfield(p, :path)..., (Symbol(x) for x in split(s, '/';keepempty=false))...)) - -const ROOT = P"/" -const LOGGER = P"/log" -const SCHEDULER = P"/scheduler" -const USER = P"/user" - -##### -# Message Processing -##### - -process(tea, args...;kw...) = tea(args...;kw...) - -##### - -function Base.put!(p::PotID, flavor) - try - put!(p[], flavor) - catch e - # TODO add test - if e isa PotNotRegisteredError - rethrow(e) - else - @error e - boil(p) - put!(p, flavor) - end - end -end - -##### - -struct CallMsg{A} - args::A - kw - promise -end - -is_prioritized(::CallMsg{<:Tuple{<:AbstractSysMsg}}) = true - -function (p::PotID)(args...;kw...) - promise = Promise(whereis(p)) # !!! the result should reside in the same place - put!(p, CallMsg(args, kw.data, promise)) - promise -end - -# ??? non specialized tea? -function process(tea, msg::CallMsg) - try - res = process(tea, msg.args...;msg.kw...) - put!(msg.promise, res) - catch err - # avoid dead lock - put!(msg.promise, err) - end -end - -##### - -function Base.:(|>)(x, p::PotID) - put!(p, x) - nothing -end - -##### - -struct GetPropMsg - prop::Symbol -end - -Base.getproperty(p::PotID, prop::Symbol) = p(GetPropMsg(prop)) - -process(tea, msg::GetPropMsg) = getproperty(tea, msg.prop) - -##### SysMsg - -""" -Signal a Pot to close the channel and release claimed resources. -By default, all children are closed recursively. -""" -struct CloseMsg <: AbstractSysMsg -end - -Base.close(p::PotID) = p(CloseMsg()) - -function process(tea, ::CloseMsg) - for c in children() - c(CloseMsg())[] - end -end - -""" -Close the active channel and remove the registered `Pot`. -""" -struct RemoveMsg <: AbstractSysMsg -end - -Base.rm(p::PotID) = p(RemoveMsg()) - -function process(tea, ::RemoveMsg) - for c in children() - c(RemoveMsg())[] - end - unregister(self()) -end - -struct ResumeMsg <: AbstractSysMsg -end - -process(tea, ::ResumeMsg) = nothing - -struct RestartMsg <: AbstractSysMsg -end - -struct PreRestartMsg <: AbstractSysMsg -end - -struct PostRestartMsg <: AbstractSysMsg -end - -##### -# Logging -##### - -Base.@kwdef struct DefaultLogger <: AbstractLogger - min_level::LogLevel = Logging.Debug - date_format::Dates.DateFormat=Dates.default_format(DateTime) -end - -Logging.shouldlog(::DefaultLogger, args...) = true -Logging.min_enabled_level(L::DefaultLogger) = L.min_level - -const DEFAULT_LOGGER = DefaultLogger() - -struct LogMsg - args - kw -end - -function (L::DefaultLogger)(msg::LogMsg) - args, kw = msg.args, msg.kw - - buf = IOBuffer() - iob = IOContext(buf, stderr) - - level, message, _module, group, id, file, line = args - - color, prefix, suffix = Logging.default_metafmt( - level, _module, group, id, file, line - ) - - printstyled(iob, "$(kw.datetime) "; color=:light_black) - printstyled(iob, prefix; bold=true, color=color) - printstyled(iob, "[$(kw.from)@$(kw.myid)]"; color=:green) - print(iob, message) - for (k,v) in pairs(kw) - if k ∉ (:datetime, :path, :myid, :from) - print(iob, " ") - printstyled(iob, k; color=:yellow) - printstyled(iob, "="; color=:light_black) - print(iob, v) - end - end - !isempty(suffix) && printstyled(iob, "($suffix)"; color=:light_black) - println(iob) - write(stderr, take!(buf)) -end - -function Logging.handle_message(logger::DefaultLogger, args...; kw...) - kw = merge( - kw.data, - ( - datetime="$(Dates.format(now(), logger.date_format))", - from=self(), - myid=myid(), - ) - ) - LogMsg(args, kw) |> LOGGER -end - -##### -# Pot Definition -##### - -struct Pot - tea_bag::Any - pid::PotID - require::RequireInfo - logger::Any -end - -function Pot( - tea_bag; - name=string(uuid4()), - cpu=eps(), - gpu=0, - logger=DEFAULT_LOGGER -) - pid = name isa PotID ? name : PotID(name) - require = RequireInfo(cpu, gpu) - Pot(tea_bag, pid, require, logger) -end - -macro pot(tea, kw...) - tea_bag = esc(:(() -> ($(tea)))) - xs = [esc(x) for x in kw] - quote - p = Pot($tea_bag; $(xs...)) - register(p) - p.pid - end -end - -struct PotState - pid::PotID - task::Task -end - -_self() = get!(task_local_storage(), KEY, PotState(USER, current_task())) -self() = _self().pid - -local_scheduler() = SCHEDULER/"local_scheduler_$(myid())" - -Base.parent() = parent(self()) -Base.parent(p::PotID) = PotID(getfield(p, :path[1:end-1])) - -children() = children(self()) - -##### -# Pot Scheduling -##### - -# TODO: set ttl? - -""" -Local cache on each worker to reduce remote call. -The links may be staled. -""" -const POT_LINK_CACHE = Dict{PotID, RemoteChannel{Channel{Any}}}() - -""" -Only valid on the driver to keep track of all registered pots. -TODO: use a kv db -""" -const POT_REGISTRY = Dict{PotID, Pot}() -const POT_CHILDREN = Dict{PotID, Set{PotID}}() - -function is_registered(p::Pot) - is_exist = remotecall_wait(1) do - haskey(Oolong.POT_REGISTRY, p.pid) - end - is_exist[] -end - -function register(p::Pot) - remotecall_wait(1) do - Oolong.POT_REGISTRY[p.pid] = p - children = get!(Oolong.POT_CHILDREN, parent(p.pid), Set{PotID}()) - push!(children, p.pid) - end -end - -function unregister(p::PotID) - remotecall_wait(1) do - delete!(Oolong.POT_REGISTRY, p) - delete!(Oolong.POT_CHILDREN, p) - end -end - -function children(p::PotID) - remotecall_wait(1) do - # ??? data race - get!(Oolong.POT_CHILDREN, p, Set{PotID}()) - end -end - -function link(p::PotID, ch::RemoteChannel) - POT_LINK_CACHE[p] = ch - if myid() != 1 - remotecall_wait(1) do - Oolong.POT_LINK_CACHE[p] = ch - end - end -end - -function Base.getindex(p::PotID) - get!(POT_LINK_CACHE, p) do - ch = remotecall_wait(1) do - get(Oolong.POT_LINK_CACHE, p, nothing) - end - if isnothing(ch[]) - boil(p) - else - ch[] - end - end -end - -whereis(p::PotID) = p[].where - -function Base.getindex(p::PotID, ::typeof(!)) - pot = remotecall_wait(1) do - get(Oolong.POT_REGISTRY, p, nothing) - end - if isnothing(pot[]) - throw(PotNotRegisteredError(p)) - else - pot[] - end -end - -""" -For debug only. Only a snapshot is returned. -!!! DO NOT MODIFY THE RESULT DIRECTLY -""" -Base.getindex(p::PotID, ::typeof(*)) = p(_self())[] - -local_boil(p::PotID) = local_boil(p[!]) - -function local_boil(p::Pot) - pid, tea_bag, logger = p.pid, p.tea_bag, p.logger - ch = RemoteChannel() do - Channel(typemax(Int),spawn=true) do ch - task_local_storage(KEY, PotState(pid, current_task())) - with_logger(logger) do - tea = tea_bag() - while true - try - flavor = take!(ch) - process(tea, flavor) - if flavor isa CloseMsg || flavor isa RemoveMsg - break - end - catch err - @debug err - flavor = parent()(err)[] - if msg isa ResumeMsg - process(tea, flavor) - elseif msg isa CloseMsg - process(tea, flavor) - break - elseif msg isa RestartMsg - process(tea, PreRestartMsg()) - tea = tea_bag() - process(tea, PostRestartMsg()) - else - @error "unknown msg received from parent: $exec" - rethrow() - end - finally - end - end - end - end - end - link(pid, ch) - ch -end - -"blocking until a valid channel is established" -boil(p::PotID) = local_scheduler()(p)[] - -struct CPUInfo - total_threads::Int - allocated_threads::Int - total_memory::Int - free_memory::Int - function CPUInfo() - new( - Sys.CPU_THREADS, - Threads.nthreads(), - convert(Int, Sys.total_memory()), - convert(Int, Sys.free_memory()), - ) - end -end - -struct GPUInfo - name::String - total_memory::Int - free_memory::Int - function GPUInfo() - new( - name(device()), - CUDA.total_memory(), - CUDA.available_memory() - ) - end -end - -struct ResourceInfo - cpu::CPUInfo - gpu::Vector{GPUInfo} -end - -function ResourceInfo() - cpu = CPUInfo() - gpu = [] - if CUDA.functional() - for d in devices() - device!(d) do - push!(gpu, GPUInfo()) - end - end - end - ResourceInfo(cpu, gpu) -end - -Base.convert(::Type{RequireInfo}, r::ResourceInfo) = RequireInfo(r.cpu.allocated_threads, length(r.gpu)) - -struct HeartBeat - resource::ResourceInfo - available::RequireInfo - from::PotID -end - -struct LocalScheduler - pending::Dict{PotID, Future} - peers::Ref{Dict{PotID, RequireInfo}} - available::Ref{RequireInfo} - timer::Timer -end - -# TODO: watch exit info - -function LocalScheduler() - pid = self() - req = convert(RequireInfo, ResourceInfo()) - available = Ref(req) - timer = Timer(1;interval=1) do t - HeartBeat(ResourceInfo(), available[], pid) |> SCHEDULER # !!! non blocking - end - - pending = Dict{PotID, Future}() - peers = Ref(Dict{PotID, RequireInfo}(pid => req)) - - LocalScheduler(pending, peers, available, timer) -end - -function (s::LocalScheduler)(p::PotID) - pot = p[!] - if pot.require <= s.available[] - res = local_boil(p) - s.available[] -= pot.require - res - else - res = Future() - s.pending[p] = res - res - end -end - -function (s::LocalScheduler)(peers::Dict{PotID, RequireInfo}) - s.peers[] = peers - for (p, f) in s.pending - pot = p[!] - for (w, r) in peers - if pot.require <= r - # transfer to w - put!(f, w(p)) - delete!(s.pending, p) - break - end - end - end -end - -Base.@kwdef struct Scheduler - workers::Dict{PotID, HeartBeat} = Dict() - pending::Dict{PotID, Future} = Dict() -end - -# ??? throttle -function (s::Scheduler)(h::HeartBeat) - # ??? TTL - s.workers[h.from] = h - - for (p, f) in s.pending - pot = p[!] - if pot.require <= h.available - put!(f, h.from(p)) - end - end - - Dict( - p => h.available - for (p, h) in s.workers - ) |> h.from # !!! non blocking -end - -# pots are all scheduled on workers only -function (s::Scheduler)(p::PotID) - pot = p[!] - for (w, h) in s.workers - if pot.require <= h.available - return w(p) - end - end - res = Future() - s.pending[p] = res - res -end - - -##### -# System Initialization -##### - -## Root - -struct Root - function Root() - local_boil(@pot DefaultLogger() name=LOGGER logger=current_logger()) - local_boil(@pot Scheduler() name=SCHEDULER) - new() - end -end - -function banner(io::IO=stdout;color=true) - c = Base.text_colors - tx = c[:normal] # text - d1 = c[:bold] * c[:blue] # first dot - d2 = c[:bold] * c[:red] # second dot - d3 = c[:bold] * c[:green] # third dot - d4 = c[:bold] * c[:magenta] # fourth dot - - if color - print(io, - """ - ____ _ | > 是非成败转头空 - / $(d1)__$(tx) \\ | | | > Success or failure, - | $(d1)| |$(tx) | ___ | | ___ _ __ __ _ | > right or wrong, - | $(d1)| |$(tx) |/ $(d2)_$(tx) \\| |/ $(d3)_$(tx) \\| '_ \\ / $(d4)_$(tx)` | | > all turn out vain. - | $(d1)|__|$(tx) | $(d2)(_)$(tx) | | $(d3)(_)$(tx) | | | | $(d4)(_)$(tx) | | - \\____/ \\___/|_|\\___/|_| |_|\\__, | | The Immortals by the River - __/ | | -- Yang Shen - |___/ | (Translated by Xu Yuanchong) - """) - else - print(io, - """ - ____ _ | > 是非成败转头空 - / __ \\ | | | > Success or failure, - | | | | ___ | | ___ _ __ __ _ | > right or wrong, - | | | |/ _ \\| |/ _ \\| '_ \\ / _` | | > all turn out vain. - | |__| | (_) | | (_) | | | | (_) | | - \\____/ \\___/|_|\\___/|_| |_|\\__, | | The Immortals by the River - __/ | | -- Yang Shen - |___/ | (Translated by Xu Yuanchong) - """) - end -end - -function start(config_file::String="Oolong.yaml";kw...) - config = nothing - if isfile(config_file) - @info "Found $config_file. Loading configs..." - config = Configurations.from_dict(Config, YAML.load_file(config_file; dicttype=Dict{String, Any});kw...) - else - @info "$config_file not found. Using default configs." - config = Config(;kw...) - end - start(config) -end - -function start(config::Config) - config.banner && banner(color=config.color) - - @info "$(@__MODULE__) starting..." - if myid() == 1 - local_boil(@pot Root() name=ROOT logger=current_logger()) - end - - if myid() in workers() - local_boil(@pot LocalScheduler() name=local_scheduler()) - end -end - -function stop() -end diff --git a/src/core/core.jl b/src/core/core.jl new file mode 100644 index 0000000..20dc60b --- /dev/null +++ b/src/core/core.jl @@ -0,0 +1,4 @@ +include("pot_id.jl") +include("pot.jl") +include("message_handling.jl") +include("scheduling.jl") \ No newline at end of file diff --git a/src/core/message_handling.jl b/src/core/message_handling.jl new file mode 100644 index 0000000..51ce506 --- /dev/null +++ b/src/core/message_handling.jl @@ -0,0 +1,136 @@ +process(tea, args...;kw...) = tea(args...;kw...) + +function Base.put!(p::PotID, flavor) + try + put!(p[], flavor) + catch e + # TODO add test + if e isa PotNotRegisteredError + rethrow(e) + else + @error e + boil(p) + put!(p, flavor) + end + end +end + +##### + +struct CallMsg{A} + args::A + kw + promise +end + +is_prioritized(::CallMsg{<:Tuple{<:AbstractSysMsg}}) = true + +function (p::PotID)(args...;kw...) + promise = Promise(whereis(p)) # !!! the result should reside in the same place + put!(p, CallMsg(args, kw.data, promise)) + promise +end + +# ??? non specialized tea? +function process(tea, msg::CallMsg) + try + res = process(tea, msg.args...;msg.kw...) + put!(msg.promise, res) + catch err + ce = CapturedException(err, catch_backtrace()) + put!(msg.promise, Failure(ce)) + rethrow(err) + end +end + +##### + +function Base.:(|>)(x, p::PotID) + put!(p, x) + nothing +end + +##### + +struct GetPropMsg + prop::Symbol +end + +Base.getproperty(p::PotID, prop::Symbol) = p(GetPropMsg(prop)) + +process(tea, msg::GetPropMsg) = getproperty(tea, msg.prop) + +##### SysMsg + +struct Exit +end + +const EXIT = Exit() + +""" + CloseWhenIdleMsg(t::Int) + +Signal a Pot and its children to close the channel and release claimed resources if the Pot has been idle for `t` seconds. +""" +struct CloseWhenIdleMsg <: AbstractSysMsg + t::Int +end + +function process(tea, msg::CloseWhenIdleMsg) + t_idle = (now() - _self().last_update) / Millisecond(1_000) + if t_idle >= msg.t && isempty(_self().ch) + for c in children() + msg |> c + end + EXIT + end +end + +##### + +""" +Close the active channel and remove the registered `Pot`. +""" +struct RemoveMsg <: AbstractSysMsg +end + +Base.rm(p::PotID) = p(RemoveMsg()) + +function process(tea, msg::RemoveMsg) + # !!! note the order + for c in children() + c(msg)[] + end + unregister(self()) + close(_self().ch) + EXIT +end + +##### + +struct ResumeMsg <: AbstractSysMsg +end + +process(tea, ::ResumeMsg) = nothing + +##### + +struct RestartMsg <: AbstractSysMsg +end + +const RESTART = RestartMsg() + +process(tea, ::RestartMsg) = RESTART + +struct PreRestartMsg <: AbstractSysMsg +end + +process(tea, ::PreRestartMsg) = nothing + +struct PostRestartMsg <: AbstractSysMsg +end + +process(tea, ::PostRestartMsg) = nothing + +process(tea, ::Failure) = RESTART + diff --git a/src/core/pot.jl b/src/core/pot.jl new file mode 100644 index 0000000..bacab5a --- /dev/null +++ b/src/core/pot.jl @@ -0,0 +1,47 @@ +struct Pot + tea_bag::Any + pid::PotID + require::ResourceInfo + logger::Any +end + +function Pot( + tea_bag; + name=string(uuid4()), + cpu=eps(), + gpu=0, + logger=DEFAULT_LOGGER +) + pid = name isa PotID ? name : PotID(name) + require = ResourceInfo(cpu, gpu) + Pot(tea_bag, pid, require, logger) +end + +macro pot(tea, kw...) + tea_bag = esc(:(() -> ($(tea)))) + xs = [esc(x) for x in kw] + quote + p = Pot($tea_bag; $(xs...)) + register(p) + p.pid + end +end + +mutable struct PotState + pid::PotID + ch::Channel + create_time::DateTime + last_update::DateTime + n_processed::UInt +end + +_self() = get!(task_local_storage(), KEY, PotState(USER, current_task())) +self() = _self().pid + +local_scheduler() = SCHEDULER/"local_scheduler_$(myid())" + +Base.parent() = parent(self()) +Base.parent(p::PotID) = PotID(getfield(p, :path[1:end-1])) + +children() = children(self()) + diff --git a/src/core/pot_id.jl b/src/core/pot_id.jl new file mode 100644 index 0000000..e28b6bb --- /dev/null +++ b/src/core/pot_id.jl @@ -0,0 +1,49 @@ +export @P_str + +struct PotID + path::Tuple{Vararg{Symbol}} +end + +""" + P"[/]your/pot/path" + +The path can be either relative or absolute path. If a relative path is provided, it will be resolved to an absolute path based on the current context. + +!!! note + We don't validate the path for you during construction. A [`PotNotRegisteredError`](@ref) will be thrown when you try to send messages to an unregistered path. +""" +macro P_str(s) + PotID(s) +end + +function Base.show(io::IO, p::PotID) + if isempty(getfield(p, :path)) + print(io, "/") + else + for x in getfield(p, :path) + print(io, '/') + print(io, x) + end + end +end + +function PotID(s::String) + if length(s) > 0 + if s[1] == '/' + PotID(Tuple(Symbol(x) for x in split(s, '/';keepempty=false))) + else + self() / s + end + else + PotID(()) + end +end + +function Base.:(/)(p::PotID, s::String) + PotID((getfield(p, :path)..., Symbol(s))) +end + +const ROOT = P"/" +const LOGGER = P"/log" +const SCHEDULER = P"/scheduler" +const USER = P"/user" diff --git a/src/core/scheduling.jl b/src/core/scheduling.jl new file mode 100644 index 0000000..e62daa5 --- /dev/null +++ b/src/core/scheduling.jl @@ -0,0 +1,272 @@ +# TODO: set ttl? + +""" +Local cache on each worker to reduce remote call. +The links may be staled. +""" +const POT_LINK_CACHE = Dict{PotID, RemoteChannel{Channel{Any}}}() + +""" +Only valid on the driver to keep track of all registered pots. +TODO: use a kv db +""" +const POT_REGISTRY = Dict{PotID, Pot}() +const POT_CHILDREN = Dict{PotID, Set{PotID}}() + +function is_registered(p::Pot) + is_exist = remotecall_wait(1) do + haskey(Oolong.POT_REGISTRY, p.pid) + end + is_exist[] +end + +function register(p::Pot) + remotecall_wait(1) do + Oolong.POT_REGISTRY[p.pid] = p + children = get!(Oolong.POT_CHILDREN, parent(p.pid), Set{PotID}()) + push!(children, p.pid) + end +end + +function unregister(p::PotID) + remotecall_wait(1) do + delete!(Oolong.POT_REGISTRY, p) + delete!(Oolong.POT_CHILDREN, p) + end +end + +function children(p::PotID) + remotecall_wait(1) do + # ??? data race + get!(Oolong.POT_CHILDREN, p, Set{PotID}()) + end +end + +function link(p::PotID, ch::RemoteChannel) + POT_LINK_CACHE[p] = ch + if myid() != 1 + remotecall_wait(1) do + Oolong.POT_LINK_CACHE[p] = ch + end + end +end + +function Base.getindex(p::PotID) + get!(POT_LINK_CACHE, p) do + ch = remotecall_wait(1) do + get(Oolong.POT_LINK_CACHE, p, nothing) + end + if isnothing(ch[]) + boil(p) + else + ch[] + end + end +end + +whereis(p::PotID) = p[].where + +function Base.getindex(p::PotID, ::typeof(!)) + pot = remotecall_wait(1) do + get(Oolong.POT_REGISTRY, p, nothing) + end + if isnothing(pot[]) + throw(PotNotRegisteredError(p)) + else + pot[] + end +end + +""" +For debug only. Only a snapshot is returned. +!!! DO NOT MODIFY THE RESULT DIRECTLY +""" +Base.getindex(p::PotID, ::typeof(*)) = p(_self())[] + +local_boil(p::PotID) = local_boil(p[!]) + +function local_boil(p::Pot) + pid, tea_bag, logger = p.pid, p.tea_bag, p.logger + ch = RemoteChannel() do + Channel(typemax(Int),spawn=true) do ch + task_local_storage(KEY, PotState(pid, current_task())) + with_logger(logger) do + tea = tea_bag() + while true + try + flavor = take!(ch) + process(tea, flavor) + if flavor isa CloseMsg || flavor isa RemoveMsg + break + end + catch err + @debug err + flavor = parent()(err)[] + if msg isa ResumeMsg + process(tea, flavor) + elseif msg isa CloseMsg + process(tea, flavor) + break + elseif msg isa RestartMsg + process(tea, PreRestartMsg()) + tea = tea_bag() + process(tea, PostRestartMsg()) + else + @error "unknown msg received from parent: $exec" + rethrow() + end + finally + end + end + end + end + end + link(pid, ch) + ch +end + +"blocking until a valid channel is established" +boil(p::PotID) = local_scheduler()(p)[!] + +struct CPUInfo + total_threads::Int + allocated_threads::Int + total_memory::Int + free_memory::Int + function CPUInfo() + new( + Sys.CPU_THREADS, + Threads.nthreads(), + convert(Int, Sys.total_memory()), + convert(Int, Sys.free_memory()), + ) + end +end + +struct GPUInfo + name::String + total_memory::Int + free_memory::Int + function GPUInfo() + new( + name(device()), + CUDA.total_memory(), + CUDA.available_memory() + ) + end +end + +struct ResourceInfo + cpu::CPUInfo + gpu::Vector{GPUInfo} +end + +function ResourceInfo() + cpu = CPUInfo() + gpu = [] + if CUDA.functional() + for d in devices() + device!(d) do + push!(gpu, GPUInfo()) + end + end + end + ResourceInfo(cpu, gpu) +end + +Base.convert(::Type{ResourceInfo}, r::ResourceInfo) = ResourceInfo(r.cpu.allocated_threads, length(r.gpu)) + +struct HeartBeat + resource::ResourceInfo + available::ResourceInfo + from::PotID +end + +struct LocalScheduler + pending::Dict{PotID, Future} + peers::Ref{Dict{PotID, ResourceInfo}} + available::Ref{ResourceInfo} + timer::Timer +end + +# TODO: watch exit info + +function LocalScheduler() + pid = self() + req = convert(ResourceInfo, ResourceInfo()) + available = Ref(req) + timer = Timer(1;interval=1) do t + HeartBeat(ResourceInfo(), available[], pid) |> SCHEDULER # !!! non blocking + end + + pending = Dict{PotID, Future}() + peers = Ref(Dict{PotID, ResourceInfo}(pid => req)) + + LocalScheduler(pending, peers, available, timer) +end + +function (s::LocalScheduler)(p::PotID) + pot = p[!] + if pot.require <= s.available[] + res = local_boil(p) + s.available[] -= pot.require + res + else + res = Future() + s.pending[p] = res + res + end +end + +function (s::LocalScheduler)(peers::Dict{PotID, ResourceInfo}) + s.peers[] = peers + for (p, f) in s.pending + pot = p[!] + for (w, r) in peers + if pot.require <= r + # transfer to w + put!(f, w(p)) + delete!(s.pending, p) + break + end + end + end +end + +Base.@kwdef struct Scheduler + workers::Dict{PotID, HeartBeat} = Dict() + pending::Dict{PotID, Future} = Dict() +end + +# ??? throttle +function (s::Scheduler)(h::HeartBeat) + # ??? TTL + s.workers[h.from] = h + + for (p, f) in s.pending + pot = p[!] + if pot.require <= h.available + put!(f, h.from(p)) + end + end + + Dict( + p => h.available + for (p, h) in s.workers + ) |> h.from # !!! non blocking +end + +# pots are all scheduled on workers only +function (s::Scheduler)(p::PotID) + pot = p[!] + for (w, h) in s.workers + if pot.require <= h.available + return w(p) + end + end + res = Future() + s.pending[p] = res + res +end + + diff --git a/src/logging.jl b/src/logging.jl new file mode 100644 index 0000000..259b07c --- /dev/null +++ b/src/logging.jl @@ -0,0 +1,148 @@ +using LoggingExtras +using LokiLogger +using Dates + +# https://github.com/JuliaLang/julia/blob/1b93d53fc4bb59350ada898038ed4de2994cce33/base/logging.jl#L142-L151 +function Base.parse(::Type{LogLevel}, s::String) + if s == string(Logging.BelowMinLevel) Logging.BelowMinLevel + elseif s == string(Logging.Debug) Logging.Debug + elseif s == string(Logging.Info) Logging.Info + elseif s == string(Logging.Warn) Logging.Warn + elseif s == string(Logging.Error) Logging.Error + elseif s == string(Logging.AboveMaxLevel) Logging.AboveMaxLevel + else + m = match(r"LogLevel\((?-?[1-9]\d*)\)", s) + if isnothing(m) + throw(ArgumentError("unknown log level")) + else + Logging.LogLevel(parse(Int, m[:level])) + end + end + +end + +function create_log_transformer(date_format) + function transformer(log) + merge( + log, + ( + datetime = Dates.format(now(), date_format), + from=self(), + myid=myid(), + ) + ) + end +end + +function create_default_fmt(with_color=false, is_expand_stack_trace=false) + function default_fmt(iob, args) + level, message, _module, group, id, file, line, kw = args + color, prefix, suffix = Logging.default_metafmt( + level, _module, group, id, file, line + ) + ignore_fields = (:datetime, :path, :myid, :from) + if with_color + printstyled(iob, "$(kw.datetime) "; color=:light_black) + printstyled(iob, prefix; bold=true, color=color) + printstyled(iob, "[$(kw.from)@$(kw.myid)]"; color=:green) + print(iob, message) + for (k,v) in pairs(kw) + if k ∉ ignore_fields + print(iob, " ") + printstyled(iob, k; color=:yellow) + printstyled(iob, "="; color=:light_black) + print(iob, v) + end + end + !isempty(suffix) && printstyled(iob, " ($suffix)"; color=:light_black) + println(iob) + else + print(iob, "$(kw.datetime) $prefix[$(kw.from)@$(kw.myid)]$message") + for (k,v) in pairs(kw) + if k ∉ ignore_fields + print(iob, " $k=$v") + end + end + !isempty(suffix) && print(iob, " ($suffix)") + println(iob) + end + end +end + +function create_logger(config::Config) + sinks = [] + + if !isnothing(config.logging.loki_logger) + push!(sinks, LokiLogger.Logger(config.logging.loki_logger.url)) + end + + if !isnothing(config.logging.driver_logger) + driver_sinks = [] + console_logger_config = config.logging.driver_logger.console_logger + if !isnothing(console_logger_config) + push!( + driver_sinks, + FormatLogger( + create_default_fmt( + config.color, + console_logger_config.is_expand_stack_trace + ) + ) + ) + end + rotating_logger_config = config.logging.driver_logger.rotating_logger + if !isnothing(rotating_logger_config) + mkpath(rotating_logger_config.path) + push!( + driver_sinks, + DatetimeRotatingFileLogger( + create_default_fmt(), + rotating_logger_config.path, + rotating_logger_config.file_format, + ) + ) + end + if isempty(driver_sinks) + push!(driver_sinks, current_logger()) + end + push!(sinks, DriverLogger(TeeLogger(driver_sinks...))) + end + + if isempty(sinks) + push!(sinks, current_logger()) + end + + TeeLogger( + ( + MinLevelLogger( + TransformerLogger( + create_log_transformer(config.logging.date_format), + s + ), + parse(Logging.LogLevel, config.logging.log_level) + ) + for s in sinks + )... + ) +end + +##### + +Base.@kwdef struct DriverLogger <: AbstractLogger + logger::TeeLogger +end + +Logging.shouldlog(::DriverLogger, args...) = true +Logging.min_enabled_level(::DriverLogger) = Logging.BelowMinLevel +Logging.catch_exceptions(::DriverLogger) = true + +struct LogMsg + args + kw +end + +Logging.handle_message(logger::DriverLogger, args...; kw...) = LogMsg(args, kw) |> LOGGER + +function (L::DriverLogger)(msg::LogMsg) + handle_message(L.logger, msg.args...;msg.kw...) +end diff --git a/src/start.jl b/src/start.jl new file mode 100644 index 0000000..a7ff638 --- /dev/null +++ b/src/start.jl @@ -0,0 +1,70 @@ +struct Root + function Root() + local_boil(@pot DefaultLogger() name=LOGGER logger=current_logger()) + local_boil(@pot Scheduler() name=SCHEDULER) + new() + end +end + +function banner(io::IO=stdout;color=true) + c = Base.text_colors + tx = c[:normal] # text + d1 = c[:bold] * c[:blue] # first dot + d2 = c[:bold] * c[:red] # second dot + d3 = c[:bold] * c[:green] # third dot + d4 = c[:bold] * c[:magenta] # fourth dot + + if color + print(io, + """ + ____ _ | > 是非成败转头空 + / $(d1)__$(tx) \\ | | | > Success or failure, + | $(d1)| |$(tx) | ___ | | ___ _ __ __ _ | > right or wrong, + | $(d1)| |$(tx) |/ $(d2)_$(tx) \\| |/ $(d3)_$(tx) \\| '_ \\ / $(d4)_$(tx)` | | > all turn out vain. + | $(d1)|__|$(tx) | $(d2)(_)$(tx) | | $(d3)(_)$(tx) | | | | $(d4)(_)$(tx) | | + \\____/ \\___/|_|\\___/|_| |_|\\__, | | The Immortals by the River + __/ | | -- Yang Shen + |___/ | (Translated by Xu Yuanchong) + """) + else + print(io, + """ + ____ _ | > 是非成败转头空 + / __ \\ | | | > Success or failure, + | | | | ___ | | ___ _ __ __ _ | > right or wrong, + | | | |/ _ \\| |/ _ \\| '_ \\ / _` | | > all turn out vain. + | |__| | (_) | | (_) | | | | (_) | | + \\____/ \\___/|_|\\___/|_| |_|\\__, | | The Immortals by the River + __/ | | -- Yang Shen + |___/ | (Translated by Xu Yuanchong) + """) + end +end + +function start(config_file::String="Oolong.yaml";kw...) + config = nothing + if isfile(config_file) + @info "Found $config_file. Loading configs..." + config = Configurations.from_dict(Config, YAML.load_file(config_file; dicttype=Dict{String, Any});kw...) + else + @info "$config_file not found. Using default configs." + config = Config(;kw...) + end + start(config) +end + +function start(config::Config) + config.banner && banner(color=config.color) + + @info "$(@__MODULE__) starting..." + if myid() == 1 + local_boil(@pot Root() name=ROOT logger=current_logger()) + end + + if myid() in workers() + local_boil(@pot LocalScheduler() name=local_scheduler()) + end +end + +function stop() +end