Skip to content

Commit

Permalink
variant of finalize for logging (#254)
Browse files Browse the repository at this point in the history 
* also log in-progress state changes
  • Loading branch information
@willscott
willscott authored Jun 25, 2021
1 parent aea3617 commit 3a15ff0
Show file tree
Hide file tree
Showing 7 changed files with 72 additions and 17 deletions.
2 changes: 1 addition & 1 deletion controller/graphql/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ func GetHandler(db state.State, accessToken string) (*http.ServeMux, error) {
if err != nil {
return nil, err
}
finishedTask, err := tsk.Finalize(p.Context, storer)
finishedTask, err := tsk.Finalize(p.Context, storer, false)
if err != nil {
return nil, err
}
Expand Down
7 changes: 4 additions & 3 deletions controller/state/statedb.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
crypto "github.com/libp2p/go-libp2p-crypto"
"github.com/multiformats/go-multicodec"
"github.com/robfig/cron/v3"
dumpjson "github.com/willscott/ipld-dumpjson"

// DB interfaces
"github.com/filecoin-project/dealbot/controller/state/postgresdb"
Expand Down Expand Up @@ -578,7 +579,7 @@ func (s *stateDB) Update(ctx context.Context, taskID string, req tasks.UpdateTas

// finish if neccesary
if updatedTask.Status == *tasks.Successful || updatedTask.Status == *tasks.Failed {
finalized, err := updatedTask.Finalize(ctx, txContextStorer(ctx, tx))
finalized, err := updatedTask.Finalize(ctx, txContextStorer(ctx, tx), false)
if err != nil {
return err
}
Expand Down Expand Up @@ -616,13 +617,13 @@ func (s *stateDB) Update(ctx context.Context, taskID string, req tasks.UpdateTas
}

func (s *stateDB) log(ctx context.Context, task tasks.Task, tx *sql.Tx) {
finalized, err := task.Finalize(ctx, txContextStorer(ctx, tx))
finalized, err := task.Finalize(ctx, txContextStorer(ctx, tx), true)
if err != nil {
return
}

taskBytes := bytes.Buffer{}
if err := dagjson.Encoder(finalized, &taskBytes); err != nil {
if err := dumpjson.Encode(finalized, &taskBytes); err != nil {
return
}
var rawJSON json.RawMessage
Expand Down
2 changes: 1 addition & 1 deletion controller/tasks.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -302,7 +302,7 @@ func (c *Controller) getTaskHandler(w http.ResponseWriter, r *http.Request) {
nilStore := func(_ ipld.LinkContext) (io.Writer, ipld.StoreCommitter, error) {
return io.Discard, func(_ ipld.Link) error { return nil }, nil
}
finished, err := task.Finalize(r.Context(), nilStore)
finished, err := task.Finalize(r.Context(), nilStore, true)
if err != nil {
log.Errorw("finalize task error", "err", err.Error())
w.WriteHeader(http.StatusInternalServerError)
Expand Down
4 changes: 4 additions & 0 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ require (
github.com/filecoin-project/go-jsonrpc v0.1.4-0.20210217175800-45ea43ac2bec
github.com/filecoin-project/go-state-types v0.1.1-0.20210506134452-99b279731c48
github.com/filecoin-project/lotus v1.9.1-0.20210607161144-fadc79a4875b
github.com/frankban/quicktest v1.11.3 // indirect
github.com/golang-migrate/migrate/v4 v4.14.2-0.20210511063805-2e7358e012a6
github.com/golang/mock v1.5.0
github.com/google/uuid v1.1.2
Expand All @@ -30,10 +31,13 @@ require (
github.com/mitchellh/go-homedir v1.1.0
github.com/multiformats/go-multiaddr v0.3.1
github.com/multiformats/go-multicodec v0.2.0
github.com/multiformats/go-multihash v0.0.15 // indirect
github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e // indirect
github.com/prometheus/client_golang v1.6.0
github.com/robfig/cron/v3 v3.0.1
github.com/stretchr/testify v1.7.0
github.com/urfave/cli/v2 v2.3.0
github.com/willscott/ipld-dumpjson v0.0.0-20210625231845-eb416d94fa54 // indirect
golang.org/x/net v0.0.0-20210119194325-5f4716e94777
modernc.org/sqlite v1.10.6
)
Expand Down
24 changes: 19 additions & 5 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -360,6 +360,7 @@ github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe
github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4=
github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20=
github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
Expand Down Expand Up @@ -470,8 +471,9 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.3 h1:x95R7cp+rSeeqAMI2knLtQ0DKlaBhv2NrtrOvafPHRo=
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M=
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
Expand Down Expand Up @@ -762,6 +764,8 @@ github.com/ipld/go-ipld-prime v0.5.1-0.20201021195245-109253e8a018/go.mod h1:0xE
github.com/ipld/go-ipld-prime v0.7.0/go.mod h1:0xEgdD6MKbZ1vF0GC+YcR/C4SQCAlRuOjIJ2i0HxqzM=
github.com/ipld/go-ipld-prime v0.7.1-0.20210519202903-3a67953d6ef3 h1:oDjTRTR4i0GenZC4bo8QvJBsZpk3bez/ipKnA/OOKyw=
github.com/ipld/go-ipld-prime v0.7.1-0.20210519202903-3a67953d6ef3/go.mod h1:0xEgdD6MKbZ1vF0GC+YcR/C4SQCAlRuOjIJ2i0HxqzM=
github.com/ipld/go-ipld-prime v0.10.0 h1:ZCd52SDUqvA3YUJEx9v2uIm1qWv6FAxBt2mhiFpoZ6s=
github.com/ipld/go-ipld-prime v0.10.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8=
github.com/ipld/go-ipld-prime-proto v0.0.0-20200428191222-c1ffdadc01e1/go.mod h1:OAV6xBmuTLsPZ+epzKkPB1e25FHk/vCtyatkdHcArLs=
github.com/ipld/go-ipld-prime-proto v0.0.0-20200922192210-9a2bfd4440a6/go.mod h1:3pHYooM9Ea65jewRwrb2u5uHZCNkNTe9ABsVB+SrkH0=
github.com/ipld/go-ipld-prime-proto v0.1.0/go.mod h1:11zp8f3sHVgIqtb/c9Kr5ZGqpnCLF1IVTNOez9TopzE=
Expand Down Expand Up @@ -874,6 +878,8 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4=
github.com/klauspost/cpuid/v2 v2.0.4 h1:g0I61F2K2DjRHz1cnxlkNSBIaePVoJIjjnHui8QHbiw=
github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
Expand Down Expand Up @@ -1268,8 +1274,9 @@ github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+
github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=
github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
Expand Down Expand Up @@ -1338,8 +1345,9 @@ github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa
github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew=
github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
github.com/multiformats/go-multihash v0.0.14 h1:QoBceQYQQtNUuf6s7wHxnE2c8bhbMqhfGzNI032se/I=
github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc=
github.com/multiformats/go-multihash v0.0.15 h1:hWOPdrNqDjwHDx82vsYGSDZNyktOJJ2dzZJzFkOV1jM=
github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg=
github.com/multiformats/go-multistream v0.0.1/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
github.com/multiformats/go-multistream v0.0.4/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg=
Expand Down Expand Up @@ -1430,8 +1438,9 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
github.com/polydawn/refmt v0.0.0-20190408063855-01bf1e26dd14/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a h1:hjZfReYVLbqFkAtr2us7vdy04YWz3LVAirzP7reh8+M=
github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e h1:ZOcivgkkFRnjfoTcGsDq3UQYiBmekwLA+qg0OjyB/ls=
github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
Expand Down Expand Up @@ -1658,6 +1667,10 @@ github.com/whyrusleeping/pubsub v0.0.0-20190708150250-92bcb0691325/go.mod h1:g7c
github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow=
github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg=
github.com/whyrusleeping/yamux v1.1.5/go.mod h1:E8LnQQ8HKx5KD29HZFUwM1PxCOdPRzGwur1mcYhXcD8=
github.com/willscott/ipld-dumpjson v0.0.0-20210625231313-f7ab2f5a7f2e h1:zFvAY/Zt+KF7atXCwsDgzeebB2P6ZW79Hzg0CJ1lqTY=
github.com/willscott/ipld-dumpjson v0.0.0-20210625231313-f7ab2f5a7f2e/go.mod h1:usOp07XD2INgfRf5TLS5erZbdeNoHEUyOSmlaOn8+5Y=
github.com/willscott/ipld-dumpjson v0.0.0-20210625231845-eb416d94fa54 h1:KN54M/NWqUZkMysljJorBXgTISTqzNtOeMxxu2Z0q0w=
github.com/willscott/ipld-dumpjson v0.0.0-20210625231845-eb416d94fa54/go.mod h1:J46otxTrmFl38OFRk/1iUZ28rppPYFhe3Vd606QhE4Q=
github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE=
github.com/xanzy/go-gitlab v0.15.0/go.mod h1:8zdQa/ri1dfn8eS3Ir1SyfvOKlw7WBJ8DVThkpGiXrs=
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
Expand Down Expand Up @@ -1958,8 +1971,9 @@ golang.org/x/sys v0.0.0-20200926100807-9d91bd62050c/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201126233918-771906719818/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210309074719-68d13333faf2 h1:46ULzRKLh1CwgRq2dC5SlBzEqqNCi8rreOZnNrbqcIY=
golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
Expand Down
30 changes: 30 additions & 0 deletions tasks/locallink.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package tasks

import (
"context"
"fmt"

ipld "github.com/ipld/go-ipld-prime"
)

type localLink struct {
ipld.Node
}

var _ ipld.Link = (*localLink)(nil)

func (l *localLink) LinkBuilder() ipld.LinkBuilder {
return nil
}

func (l *localLink) Load(_ context.Context, _ ipld.LinkContext, na ipld.NodeAssembler, _ ipld.Loader) error {
return na.AssignNode(l.Node)
}

func (l *localLink) String() string {
return fmt.Sprintf("&%s", l.Node)
}

func (l *localLink) Representation() ipld.Node {
return l.Node
}
20 changes: 13 additions & 7 deletions tasks/model.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -344,9 +344,11 @@ func (t *_Task) Assign(worker string, status Status) Task {
return &newTask
}

func (t *_Task) Finalize(ctx context.Context, s ipld.Storer) (FinishedTask, error) {
if t.Status != *Failed && t.Status != *Successful {
return nil, fmt.Errorf("task cannot be finalized as it is not in a finished state")
func (t *_Task) Finalize(ctx context.Context, s ipld.Storer, local bool) (FinishedTask, error) {
if !local {
if t.Status != *Failed && t.Status != *Successful {
return nil, fmt.Errorf("task cannot be finalized as it is not in a finished state")
}
}

logs := parseFinalLogs(t)
Expand All @@ -372,11 +374,15 @@ func (t *_Task) Finalize(ctx context.Context, s ipld.Storer) (FinishedTask, erro
if t.PastStageDetails.Exists() {
logList = t.PastStageDetails.Must()
}
logLnk, err := linkProto.Build(ctx, ipld.LinkContext{}, logList, s)
if err != nil {
return nil, err
if local {
ft.Events = _Link_List_StageDetails{&localLink{logList}}
} else {
logLnk, err := linkProto.Build(ctx, ipld.LinkContext{}, logList, s)
if err != nil {
return nil, err
}
ft.Events = _Link_List_StageDetails{logLnk}
}
ft.Events = _Link_List_StageDetails{logLnk}

return &ft, nil
}
Expand Down

0 comments on commit 3a15ff0

Please sign in to comment.