Skip to content

Commit

Permalink
feat/daemon spawn (#240)
Browse files Browse the repository at this point in the history
* stub daemon flags

* regions controller

* daemons api

* local exec spawner

* tidy

* kubernetes spawner constructor

* use templating

* default in view func

* output daemon on create

* templating

* remove wallets api

* use static slice length on Get

* use lock on read-operations List and Get

* use sigs yaml

* bug, wallet nil pointer

* remove extra log info

* tags

* include linux headers in docker file
  • Loading branch information
Cory Schwartz authored Jul 8, 2021
1 parent e400dc9 commit cff66ff
Show file tree
Hide file tree
Showing 12 changed files with 1,077 additions and 13 deletions.
3 changes: 2 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,13 @@
FROM node:14-alpine AS js
WORKDIR /usr/src/app
COPY ./controller/app .
run apk add npm
RUN npm install

FROM golang:alpine AS builder
RUN apk update
RUN apk upgrade
RUN apk add --update gcc>=9.3.0 g++>=9.3.0 alpine-sdk
RUN apk add --update gcc>=9.3.0 g++>=9.3.0 alpine-sdk linux-headers

WORKDIR /go/src/app/

Expand Down
6 changes: 6 additions & 0 deletions commands/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,7 @@ var ControllerFlags = []cli.Flag{
Usage: "host:port to bind http server on",
Aliases: []string{"l"},
EnvVars: []string{"DEALBOT_LISTEN"},
Value: "127.0.0.1:33258",
}),
altsrc.NewStringFlag(&cli.StringFlag{
Name: "graphql",
Expand Down Expand Up @@ -266,6 +267,11 @@ var ControllerFlags = []cli.Flag{
Usage: "set an access secret for access to inprogress data over gql",
EnvVars: []string{"DEALBOT_GRAPHQL_ACCESS_TOKEN"},
}),
altsrc.NewStringFlag(&cli.StringFlag{
Name: "daemon-driver",
Usage: "system to use when starting daemons",
EnvVars: []string{"DEALBOT_DAEMON_DRIVER"},
}),
altsrc.NewStringFlag(&cli.StringFlag{
Name: "devAssetDir",
Usage: "build frontend assets from directory instead of embedded version (set to location of 'controller'; the directory containing static and app)",
Expand Down
12 changes: 11 additions & 1 deletion controller/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"time"

"github.com/filecoin-project/dealbot/controller/graphql"
"github.com/filecoin-project/dealbot/controller/spawn"
"github.com/filecoin-project/dealbot/controller/state"
"github.com/filecoin-project/dealbot/controller/webutil"
"github.com/filecoin-project/dealbot/metrics"
Expand Down Expand Up @@ -48,6 +49,7 @@ type Controller struct {
db state.State
basicauth string
metricsRecorder metrics.MetricsRecorder
spawner spawn.Spawner
}

func New(ctx *cli.Context) (*Controller, error) {
Expand Down Expand Up @@ -114,7 +116,6 @@ func New(ctx *cli.Context) (*Controller, error) {
if err != nil {
return nil, err
}

return NewWithDependencies(ctx, l, gl, recorder, backend)
}

Expand All @@ -134,6 +135,11 @@ func NewWithDependencies(ctx *cli.Context, listener, graphqlListener net.Listene
srv := new(Controller)
srv.db = backend
srv.basicauth = ctx.String("basicauth")
if ctx.String("daemon-driver") == "kubernetes" {
srv.spawner = spawn.NewKubernetes()
} else {
srv.spawner = spawn.NewLocal(ctx.String("listen"))
}

r := mux.NewRouter().StrictSlash(true)

Expand Down Expand Up @@ -162,6 +168,10 @@ func NewWithDependencies(ctx *cli.Context, listener, graphqlListener net.Listene
r.HandleFunc("/tasks/{uuid}", srv.deleteTaskHandler).Methods("DELETE")
r.HandleFunc("/car", srv.carHandler).Methods("GET")
r.HandleFunc("/health", srv.healthHandler).Methods("GET")
r.HandleFunc("/regions", srv.getRegionsHandler).Methods("GET")
r.HandleFunc("/regions/{regionid}", srv.getDaemonsHandler).Methods("GET")
r.HandleFunc("/regions/{regionid}", srv.newDaemonHandler).Methods("POST")
r.HandleFunc("/regions/{regionid}/{daemonid}", srv.getDaemonHandler).Methods("GET")
r.HandleFunc("/cred.js", srv.authHandler).Methods("GET")
r.Methods("OPTIONS").HandlerFunc(srv.sendCORSHeaders)
metricsHandler := recorder.Handler()
Expand Down
117 changes: 117 additions & 0 deletions controller/daemons.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
package controller

import (
"encoding/hex"
"encoding/json"
"errors"
"net/http"

"github.com/filecoin-project/dealbot/controller/spawn"
"github.com/google/uuid"

"github.com/filecoin-project/lotus/chain/types"
"github.com/filecoin-project/lotus/chain/wallet"
_ "github.com/filecoin-project/lotus/lib/sigs/bls"
_ "github.com/filecoin-project/lotus/lib/sigs/secp"

"github.com/gorilla/mux"
)

type DaemonList struct {
Daemons []*spawn.Daemon `json:"daemons"`
}

func (c *Controller) getDaemonsHandler(w http.ResponseWriter, r *http.Request) {
logger := log.With("req_id", r.Header.Get("X-Request-ID"))
logger.Debugw("handle request", "command", "list tasks")
defer logger.Debugw("request handled", "command", "list tasks")
w.Header().Set("Content-Type", "application/json")
enableCors(&w, r)

regionid := mux.Vars(r)["regionid"]
daemons, err := c.spawner.List(regionid)
if err != nil {
if errors.Is(err, spawn.RegionNotFound) {
w.WriteHeader(http.StatusNotFound)
} else {
log.Errorw("error listing region", "err", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
json.NewEncoder(w).Encode(&DaemonList{
Daemons: daemons,
})
}

func (c *Controller) getDaemonHandler(w http.ResponseWriter, r *http.Request) {
logger := log.With("req_id", r.Header.Get("X-Request-ID"))
logger.Debugw("handle request", "command", "list tasks")
defer logger.Debugw("request handled", "command", "list tasks")
w.Header().Set("Content-Type", "application/json")
enableCors(&w, r)

regionid := mux.Vars(r)["regionid"]
daemonid := mux.Vars(r)["daemonid"]
daemon, err := c.spawner.Get(regionid, daemonid)
if err != nil {
if errors.Is(err, spawn.DaemonNotFound) {
w.WriteHeader(http.StatusNotFound)
} else {
log.Errorw("error getting daemon", "err", err)
w.WriteHeader(http.StatusInternalServerError)
}
return
}
json.NewEncoder(w).Encode(daemon)
}

func (c *Controller) newDaemonHandler(w http.ResponseWriter, r *http.Request) {
logger := log.With("req_id", r.Header.Get("X-Request-ID"))
logger.Debugw("handle request", "command", "list tasks")
defer logger.Debugw("request handled", "command", "list tasks")
w.Header().Set("Content-Type", "application/json")
enableCors(&w, r)

daemon := new(spawn.Daemon)
if err := json.NewDecoder(r.Body).Decode(daemon); err != nil {
log.Info("could not decode daemon request", "err", err)
w.WriteHeader(http.StatusBadRequest)
return
}
regionid := mux.Vars(r)["regionid"]
daemon.Region = regionid
// generate random values if they aren't already setup
daemonDefaults(daemon)
if err := c.spawner.Spawn(daemon); err != nil {
log.Errorw("could not spawn daemon", "daemonid", daemon.Id, "err", err)
w.WriteHeader(http.StatusInternalServerError)
return
}
// daemon is already created; sanatize output
daemon.Wallet.Exported = ""
json.NewEncoder(w).Encode(daemon)
}

func daemonDefaults(d *spawn.Daemon) {
if !(d.Wallet != nil && d.Wallet.Address != "" && d.Wallet.Exported != "") {
k, _ := wallet.GenerateKey(types.KTSecp256k1)
b, _ := json.Marshal(k.KeyInfo)
d.Wallet = &spawn.Wallet{
Address: k.Address.String(),
Exported: hex.EncodeToString(b),
}
}
if d.Id == "" {
d.Id = "daemon-" + uuid.New().String()[:8]
}
if d.Workers == 0 {
d.Workers = 1
}
if d.DockerRepo == "" {
d.DockerRepo = "filecoin/dealbot"
}
if d.DockerTag == "" {
d.DockerTag = "latest"
}
}
25 changes: 25 additions & 0 deletions controller/regions.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package controller

import (
"encoding/json"
"net/http"
)

type RegionList struct {
Regions []string `json:"regions"`
}

func (c *Controller) getRegionsHandler(w http.ResponseWriter, r *http.Request) {
logger := log.With("req_id", r.Header.Get("X-Request-ID"))

logger.Debugw("handle request", "command", "list tasks")
defer logger.Debugw("request handled", "command", "list tasks")

w.Header().Set("Content-Type", "application/json")

enableCors(&w, r)

json.NewEncoder(w).Encode(&RegionList{
Regions: c.spawner.Regions(),
})
}
Loading

0 comments on commit cff66ff

Please sign in to comment.