diff --git a/.gitignore b/.gitignore index 1fa6eb99..b3895239 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,8 @@ dealbot.toml *~ +controller/static/script.js +node_modules + # Binaries for programs and plugins *.exe diff --git a/Dockerfile b/Dockerfile index 336356e2..68513c34 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,9 @@ +# run npm install in a container with node. +FROM node:14-alpine AS js +WORKDIR /usr/src/app +COPY ./controller/app . +RUN npm install + FROM golang:alpine AS builder RUN apk update RUN apk upgrade @@ -6,8 +12,10 @@ RUN apk add --update gcc>=9.3.0 g++>=9.3.0 alpine-sdk WORKDIR /go/src/app/ COPY . . +COPY --from=js /usr/src/app ./controller/app # Fetch dependencies. RUN go get -d -v ./... +RUN go generate ./... RUN go build -o dealbot -ldflags "-X github.com/filecoin-project/dealbot/controller.buildDate=`date -u +%d/%m/%Y@%H:%M:%S`" ./ FROM alpine @@ -16,4 +24,4 @@ COPY --from=builder /go/src/app/dealbot /dealbot ENV DEALBOT_LOG_JSON=true ENV DEALBOT_WORKERS=10 ENV STAGE_TIMEOUT=DefaultStorage=48h,DefaultRetrieval=48h -ENTRYPOINT ["/dealbot"] \ No newline at end of file +ENTRYPOINT ["/dealbot"] diff --git a/commands/flags.go b/commands/flags.go index 95ee1978..59a26ac1 100644 --- a/commands/flags.go +++ b/commands/flags.go @@ -261,6 +261,10 @@ var ControllerFlags = []cli.Flag{ Usage: "set an access secret for access to inprogress data over gql", EnvVars: []string{"DEALBOT_GRAPHQL_ACCESS_TOKEN"}, }), + altsrc.NewStringFlag(&cli.StringFlag{ + Name: "devAssetDir", + Usage: "build frontend assets from directory instead of embedded version (set to location of 'controller'; the directory containing static and app)", + }), } var AllFlags = append(DealFlags, append(SingleTaskFlags, append(DaemonFlags, append(ControllerFlags, MockFlags...)...)...)...) diff --git a/controller/app/index.js b/controller/app/index.js new file mode 100644 index 00000000..792f0a28 --- /dev/null +++ b/controller/app/index.js @@ -0,0 +1,94 @@ +import "./jquery-global"; +import "bootstrap-cron-picker/dist/cron-picker"; + +$().ready(() => { + $('#newSchedule').cronPicker(); + + $("#addDone").hide(); + if ($('#newSR').is(':checked')) { + $("#newstorage").hide(); + } else { + $("#newretrieval").hide(); + } + $("#newSR").on('change', () => { + if ($('#newSR').is(':checked')) { + $("#newretrieval").show(); + $("#newstorage").hide(); + } else { + $("#newretrieval").hide(); + $("#newstorage").show(); + } + }) + + if (!$('#newRepeat').is(':checked')) { + $("#setschedule").hide(); + } + $("#newRepeat").on('change', () => { + if ($('#newRepeat').is(':checked')) { + $("#setschedule").show(); + } else { + $("#setschedule").hide(); + } + }) + + $("#addtask button").on('click', doSubmit); + $("schedulesection form").on('submit', doSubmit); +}) + +function doSubmit(e) { + if (e.preventDefault) { + e.preventDefault() + } + + // loop over miners + let miners = $("#newMiner").val().trim().split('\n') + + let remaining = miners.length; + let done = () => { + remaining--; + if (!remaining) { + $("#addDone").show() + } + } + + for (let i = 0; i < miners.length; i++) { + let miner = miners[i]; + let url = "/tasks/storage"; + let data = {}; + if ($('#newSR').is(':checked')) { + url = "/tasks/retrieval"; + data = { + "Miner": miner, + "PayloadCID": $('#newCid').val(), + "CARExport": false, + "MaxPriceAttoFIL": 20000000000, + } + } else { + data = { + "Miner": miner, + "Size": $('#newSize').val(), + "StartOffset": 6152, // 3 days? + "FastRetrieval": $('#newFast').is(':checked'), + "Verified": $('#newVerified').is(':checked'), + "MaxPriceAttoFIL": 20000000000, + } + } + + if ($('#newRepeat').is(':checked')) { + data.Schedule = $('#newSchedule').val() + data.ScheduleLimit = $('#newScheduleLimit').val() + } + if ($('#newScheduleTag').val() !='') { + data.Tag = $('#newScheduleTag').val() + } + + $.ajax({ + type: "POST", + url: url, + data: data, + success: done, + }); + } + + return false +} diff --git a/controller/app/jquery-global.js b/controller/app/jquery-global.js new file mode 100644 index 00000000..eb71c691 --- /dev/null +++ b/controller/app/jquery-global.js @@ -0,0 +1,3 @@ +import jquery from 'jquery'; +window.jQuery = jquery; +window.$ = jquery; diff --git a/controller/app/package-lock.json b/controller/app/package-lock.json new file mode 100644 index 00000000..91de01ca --- /dev/null +++ b/controller/app/package-lock.json @@ -0,0 +1,30 @@ +{ + "name": "dealbot-controller", + "version": "1.0.0", + "lockfileVersion": 1, + "requires": true, + "dependencies": { + "bootstrap": { + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/bootstrap/-/bootstrap-3.4.1.tgz", + "integrity": "sha512-yN5oZVmRCwe5aKwzRj6736nSmKDX7pLYwsXiCj/EYmo16hODaBiT4En5btW/jhBF/seV+XMx3aYwukYC3A49DA==", + "dev": true + }, + "bootstrap-cron-picker": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/bootstrap-cron-picker/-/bootstrap-cron-picker-1.0.0.tgz", + "integrity": "sha1-fiTVFfLHWXgsmyr0mFVzDGlSYdE=", + "dev": true, + "requires": { + "bootstrap": "^3.3.7", + "jquery": "^3.2.1" + } + }, + "jquery": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/jquery/-/jquery-3.6.0.tgz", + "integrity": "sha512-JVzAR/AjBvVt2BmYhxRCSYysDsPcssdmTFnzyLEts9qNwmjmu4JTAMYubEfwVOSwpQ1I1sKKFcxhZCI2buerfw==", + "dev": true + } + } +} diff --git a/controller/app/package.json b/controller/app/package.json new file mode 100644 index 00000000..b87ff3e7 --- /dev/null +++ b/controller/app/package.json @@ -0,0 +1,23 @@ +{ + "name": "dealbot-controller", + "version": "1.0.0", + "description": "dealbot controller webapp logic", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "git+https://github.com/filecoin-project/dealbot.git" + }, + "author": "Protocol Labs", + "license": "MIT", + "bugs": { + "url": "https://github.com/filecoin-project/dealbot/issues" + }, + "homepage": "https://github.com/filecoin-project/dealbot#readme", + "devDependencies": { + "bootstrap-cron-picker": "^1.0.0", + "jquery": "^3.6.0" + } +} diff --git a/controller/controller.go b/controller/controller.go index 4db3b69a..7efbbd2e 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -1,18 +1,23 @@ +//go:generate go run ./webutil/gen app static/script.js + package controller import ( "context" "embed" "fmt" + "io" "io/fs" "io/ioutil" "net" "net/http" "os" + "path" "time" "github.com/filecoin-project/dealbot/controller/graphql" "github.com/filecoin-project/dealbot/controller/state" + "github.com/filecoin-project/dealbot/controller/webutil" "github.com/filecoin-project/dealbot/metrics" metricslog "github.com/filecoin-project/dealbot/metrics/log" "github.com/filecoin-project/dealbot/metrics/prometheus" @@ -101,12 +106,7 @@ func New(ctx *cli.Context) (*Controller, error) { return nil, err } - gqlToken := "" - if ctx.IsSet("gqlAccessToken") { - gqlToken = ctx.String("gqlAccessToken") - } - - return NewWithDependencies(l, gl, gqlToken, recorder, backend) + return NewWithDependencies(ctx, l, gl, recorder, backend) } type logEcapsulator struct { @@ -121,7 +121,7 @@ func (fw *logEcapsulator) Write(p []byte) (n int, err error) { //go:embed static var static embed.FS -func NewWithDependencies(listener, graphqlListener net.Listener, gqlToken string, recorder metrics.MetricsRecorder, backend state.State) (*Controller, error) { +func NewWithDependencies(ctx *cli.Context, listener, graphqlListener net.Listener, recorder metrics.MetricsRecorder, backend state.State) (*Controller, error) { srv := new(Controller) srv.db = backend @@ -156,7 +156,18 @@ func NewWithDependencies(listener, graphqlListener net.Listener, gqlToken string if metricsHandler != nil { r.Handle("/metrics", metricsHandler) } - r.PathPrefix("/").Handler(http.FileServer(http.FS(statDir))) + + if ctx.IsSet("devAssetDir") { + scriptResolver := func(w http.ResponseWriter, r *http.Request) { + data := webutil.Compile(path.Join(ctx.String("devAssetDir"), "app"), false) + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, data) + } + r.HandleFunc("/script.js", scriptResolver) + r.PathPrefix("/").Handler(http.FileServer(http.Dir(path.Join(ctx.String("devAssetDir"), "static")))) + } else { + r.PathPrefix("/").Handler(http.FileServer(http.FS(statDir))) + } srv.doneCh = make(chan struct{}) srv.server = &http.Server{ @@ -166,7 +177,7 @@ func NewWithDependencies(listener, graphqlListener net.Listener, gqlToken string } if graphqlListener != nil { - gqlHandler, err := graphql.GetHandler(srv.db, gqlToken) + gqlHandler, err := graphql.GetHandler(srv.db, ctx.String("gqlAccessToken")) if err != nil { return nil, err } diff --git a/controller/http_test.go b/controller/http_test.go index 208514e3..fe8bfe4c 100644 --- a/controller/http_test.go +++ b/controller/http_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "flag" "fmt" "io" "io/ioutil" @@ -21,6 +22,7 @@ import ( "github.com/ipld/go-ipld-prime/codec/dagjson" "github.com/libp2p/go-libp2p-core/crypto" "github.com/stretchr/testify/require" + "github.com/urfave/cli/v2" ) const jsonTestDeals = "../devnet/sample_tasks.json" @@ -316,7 +318,8 @@ func newHarness(ctx context.Context, t *testing.T) *harness { require.NoError(t, err) be, err := state.NewStateDB(ctx, "sqlite", h.dbloc+"/tmp.sqlite", pr, h.recorder) require.NoError(t, err) - h.controller, err = controller.NewWithDependencies(listener, nil, "", h.recorder, be) + cc := cli.NewContext(cli.NewApp(), &flag.FlagSet{}, nil) + h.controller, err = controller.NewWithDependencies(cc, listener, nil, h.recorder, be) h.serveErr = make(chan error, 1) go func() { diff --git a/controller/state/statedb.go b/controller/state/statedb.go index 508ee601..c9ef6ac6 100644 --- a/controller/state/statedb.go +++ b/controller/state/statedb.go @@ -877,7 +877,7 @@ func (s *stateDB) PublishRecordsFrom(ctx context.Context, worker string) error { if err != nil { return err } - tskBuilder := tasks.Type.FinishedTask.NewBuilder() + tskBuilder := tasks.Type.FinishedTask__Repr.NewBuilder() if err := dagjson.Decoder(tskBuilder, rcrdRdr); err != nil { return err } diff --git a/controller/static/index.html b/controller/static/index.html index aae7800d..176feaa6 100644 --- a/controller/static/index.html +++ b/controller/static/index.html @@ -1,22 +1,133 @@ - + + Dealbot Controller - + + + - +

Dealbot

-
-
Retrieval Ingest
-
-
- -
- -
-
+
+
+

+ +

+
+
+ List of tasks and ability to cancel them. + + + + + + + + + +
IDStatusTaskSchedule
+
+
+
+
+

+ +

+
+
+ Schedule storage and retrieval tasks +
+
+ + +
+ + +
+ + +
+ +
+
+ + +
+
+ + +
+
+ + +
+
+
+
+ + +
+ +
+
+
+ + +
+
+ +
+
+ + +
+
+
+ Restrict scheduling to dealbots with tag: + +
+
+ + +
+
+
+
+
+
+

+ +

+
+
+ List of bots and ability to spin up / shutdown. +
+ This section is not complete yet. +
+
+
+ +
+ + - \ No newline at end of file + diff --git a/controller/static/script.js b/controller/static/script.js deleted file mode 100644 index 098b23a6..00000000 --- a/controller/static/script.js +++ /dev/null @@ -1,23 +0,0 @@ -async function onSubmit(evt) { - evt.preventDefault(); - - let itms = document.getElementById('retdat').value.trim().split(/\s+/); - for (;itms.length>0;) { - let miner = itms.shift(); - let cid = itms.shift(); - let resp = await fetch('/tasks/retrieval', { - method: 'POST', - headers: {'Content-Type': 'application/json'}, - body: JSON.stringify({"Miner":miner,"PayloadCID":cid,"CARExport":false}) - }) - } - console.log(resp) - - return false; -} - -function setupForm() { - document.getElementsByTagName('form')[0].addEventListener('submit', onSubmit, true); -} - -window.addEventListener('load',setupForm, true); \ No newline at end of file diff --git a/controller/static/style.css b/controller/static/style.css new file mode 100644 index 00000000..791a65aa --- /dev/null +++ b/controller/static/style.css @@ -0,0 +1,51 @@ +.cron-picker-dow { + margin-bottom: 10px; +} +.cron-picker-dow button { + margin-right: 3px; +} + +.cron-picker-recurrence-types { + margin-bottom: 10px; +} + +.cron-picker-recurrence-types li { + margin-right: 5px; +} + +.cron-picker-recurrence-types .active { + font-weight: bold; + text-decoration: underline; +} + +.hidden { + display:none; +} + +.cron-picker-day-filter { + margin-bottom: 10px; +} + +.cron-picker-day-filter .cron-picker-day-type-filter, .cron-picker-weekday-type-filter { + display: inline-block; +} + +.cron-picker-day-filter select { + display: inline-block; + width: 60px; + margin: 0 5px; +} + +.cron-picker-day-filter .cron-picker-dow-select { + width: 100px; +} +.cron-picker-dow button.active { + background: #0c63e4; +} + +.cron-picker-time select { + width: 60px; + display: inline-block; + margin: 0 5px; +} + diff --git a/controller/webutil/build.go b/controller/webutil/build.go new file mode 100644 index 00000000..a584666c --- /dev/null +++ b/controller/webutil/build.go @@ -0,0 +1,28 @@ +package webutil + +import ( + "path" + + "github.com/evanw/esbuild/pkg/api" +) + +// Compile executes esbuild to bundle client-side app logic +func Compile(rootPath string, minify bool) string { + opts := api.BuildOptions{ + EntryPoints: []string{path.Join(rootPath, "index.js")}, + Outfile: "script.js", + Bundle: true, + Write: false, + LogLevel: api.LogLevelInfo, + } + if minify { + opts.MinifyWhitespace = true + opts.MinifyIdentifiers = false + opts.MinifySyntax = true + } + res := api.Build(opts) + if len(res.Errors) > 0 { + return "" + } + return string(res.OutputFiles[0].Contents) +} diff --git a/controller/webutil/gen/gen.go b/controller/webutil/gen/gen.go new file mode 100644 index 00000000..6d73b540 --- /dev/null +++ b/controller/webutil/gen/gen.go @@ -0,0 +1,38 @@ +package main + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path" + + "github.com/filecoin-project/dealbot/controller/webutil" +) + +func main() { + if len(os.Args) < 2 { + fmt.Printf("Must specify source directory") + os.Exit(1) + } + + cmd := exec.Command("npm", "install") + cmd.Dir = os.Args[1] + err := cmd.Run() + if err != nil { + fmt.Printf("Failed to install frontend dependencies: %v\n", err) + if _, err := os.Stat(path.Join(os.Args[1], "node_modules")); os.IsNotExist(err) { + os.Exit(1) + } + } + + data := webutil.Compile(os.Args[1], true) + if len(os.Args) < 3 { + fmt.Printf("%s\n", data) + os.Exit(0) + } + if err := ioutil.WriteFile(os.Args[2], []byte(data), 0644); err != nil { + fmt.Printf("%s\n", err) + os.Exit(1) + } +} diff --git a/go.mod b/go.mod index 98d9a362..9283b157 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.16 require ( github.com/c2h5oh/datasize v0.0.0-20200825124411-48ed595a09d2 + github.com/evanw/esbuild v0.12.9 github.com/filecoin-project/go-address v0.0.5 github.com/filecoin-project/go-fil-markets v1.4.0 github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec diff --git a/go.sum b/go.sum index c1828581..0e4ec608 100644 --- a/go.sum +++ b/go.sum @@ -264,6 +264,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etclabscore/go-jsonschema-walk v0.0.6/go.mod h1:VdfDY72AFAiUhy0ZXEaWSpveGjMT5JcDIm903NGqFwQ= github.com/etclabscore/go-openrpc-reflect v0.0.36/go.mod h1:0404Ky3igAasAOpyj1eESjstTyneBAIk5PgJFbK4s5E= +github.com/evanw/esbuild v0.12.9 h1:5AqZCmKyew2uWhbHAOnJJoxk7l2h05V+mTHyEBAQqbk= +github.com/evanw/esbuild v0.12.9/go.mod h1:y2AFBAGVelPqPodpdtxWWqe6n2jYf5FrsJbligmRmuw= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 h1:BBso6MBKW8ncyZLv37o+KNyy0HrrHgfnOaGQC2qvN+A= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5/go.mod h1:JpoxHjuQauoxiFMl1ie8Xc/7TfLuMZ5eOCONd1sUBHg= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= @@ -1943,6 +1945,7 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501145240-bc7a7d42d5c3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200509044756-6aff5f38e54f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=