From 767d2ad93b5b45d74fea5b35c6c9e64a1b50e9e9 Mon Sep 17 00:00:00 2001 From: b5 Date: Sat, 14 Sep 2019 23:17:43 -0400 Subject: [PATCH] fix(add): need to re-initialize add when connecting if we have an IPFS node instance, node.GoOnline has to make a new instance to connect properly. If remoteClient retains the reference to the old instance, we run into issues where the online instance can't "see" the additions. We fix that by re-initializing the client with the new instance. Debugging this one was not fun. --- actions/dataset.go | 9 ++++++--- api/api.go | 5 ++--- base/dataset.go | 13 +++++++++++-- go.mod | 5 ++--- go.sum | 19 +++++++++++++++++++ lib/lib.go | 21 +++++++++++++++++++++ p2p/node.go | 7 +++++++ remote/client.go | 18 ++++++++++++++---- 8 files changed, 82 insertions(+), 15 deletions(-) diff --git a/actions/dataset.go b/actions/dataset.go index 7dc8e9b13..97516af99 100644 --- a/actions/dataset.go +++ b/actions/dataset.go @@ -151,6 +151,8 @@ func AddDataset(ctx context.Context, node *p2p.QriNode, rc *remote.Client, remot Error error } + fetchCtx, cancelFetch := context.WithCancel(ctx) + defer cancelFetch() responses := make(chan addResponse) tasks := 0 @@ -172,13 +174,13 @@ func AddDataset(ctx context.Context, node *p2p.QriNode, rc *remote.Client, remot responses <- res }() - if err := rc.PullDataset(ctx, ref, remoteAddr); err != nil { + if err := rc.PullDataset(fetchCtx, ref, remoteAddr); err != nil { res.Error = err return } node.LocalStreams.PrintErr("🗼 fetched from registry\n") if pinner, ok := node.Repo.Store().(cafs.Pinner); ok { - err := pinner.Pin(ctx, ref.Path, true) + err := pinner.Pin(fetchCtx, ref.Path, true) res.Error = err } }(refCopy) @@ -187,7 +189,7 @@ func AddDataset(ctx context.Context, node *p2p.QriNode, rc *remote.Client, remot if node.Online { tasks++ go func() { - err := base.FetchDataset(ctx, node.Repo, ref, true, true) + err := base.FetchDataset(fetchCtx, node.Repo, ref, true, true) responses <- addResponse{ Ref: ref, Error: err, @@ -204,6 +206,7 @@ func AddDataset(ctx context.Context, node *p2p.QriNode, rc *remote.Client, remot res := <-responses err = res.Error if err == nil { + cancelFetch() success = true *ref = *res.Ref break diff --git a/api/api.go b/api/api.go index d2eb1c394..87c6b79ef 100644 --- a/api/api.go +++ b/api/api.go @@ -49,9 +49,8 @@ func (s Server) Serve(ctx context.Context) (err error) { node := s.Node() cfg := s.Config() - if err = node.GoOnline(); err != nil { - fmt.Println("serving error", cfg.P2P.Enabled) - return + if err := s.Instance.Connect(ctx); err != nil { + return err } server := &http.Server{} diff --git a/base/dataset.go b/base/dataset.go index d921efae1..55c1d2fc1 100644 --- a/base/dataset.go +++ b/base/dataset.go @@ -44,10 +44,19 @@ func OpenDataset(ctx context.Context, fsys qfs.Filesystem, ds *dataset.Dataset) return } } + + // TODO (b5) - this is an error sometimes caused by failing to properly pin the + // rendered file. this'll really trip up if we're online, b/c ipfs will hang + // forever looking for the missing render dag :() if ds.Viz != nil && ds.Viz.RenderedFile() == nil { if err = ds.Viz.OpenRenderedFile(ctx, fsys); err != nil { - log.Debug(err) - return + if strings.Contains(err.Error(), "not found") { + log.Debug("skipping not-found viz script") + err = nil + } else { + log.Debug(err) + return + } } } return diff --git a/go.mod b/go.mod index 522d3c014..222ae9bb2 100644 --- a/go.mod +++ b/go.mod @@ -56,11 +56,10 @@ require ( github.com/qri-io/varName v0.1.0 github.com/sergi/go-diff v1.0.0 github.com/sirupsen/logrus v1.2.0 - github.com/spf13/cobra v0.0.4 + github.com/spf13/cobra v0.0.5 github.com/theckman/go-flock v0.7.1 go.starlark.net v0.0.0-20190528202925-30ae18b8564f golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 - golang.org/x/net v0.0.0-20190620200207-3b0461eec859 // indirect - golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae + golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb gopkg.in/yaml.v2 v2.2.2 ) diff --git a/go.sum b/go.sum index ffb5ba030..153a87e3d 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,8 @@ github.com/360EntSecGroup-Skylar/excelize v1.4.1 h1:l55mJb6rkkaUzOpSsgEeKYtS6/0g github.com/360EntSecGroup-Skylar/excelize v1.4.1/go.mod h1:vnax29X2usfl7HHkBrX5EvSCJcmH3dT9luvxzu8iGAE= github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7 h1:PqzgE6kAMi81xWQA2QIVxjWkFHptGgC547vchpUbtFo= github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkByhMAwYaFAX9w2l7vxvBQ5NMoxDrkhqhtn4= +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/Kubuxu/gocovmerge v0.0.0-20161216165753-7ecaa51963cd/go.mod h1:bqoB8kInrTeEtYAwaIXoSRqdwnjQmFhsfusnzyui6yY= @@ -82,10 +84,13 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018 h1:6xT9KW8zLC5IlbaIF5Q7JNieBoACT7iW0YTxQHR0in0= github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4= github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= +github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger v2.0.0-rc.2+incompatible h1:7KPp6xv5+wymkVUbkAnZZXvmDrJlf09m/7u1HG5lAYA= github.com/dgraph-io/badger v2.0.0-rc.2+incompatible/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f h1:dDxpBYafY/GYpcl+LS4Bn3ziLPuEdGRkRjYAbSlWxSA= github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -202,6 +207,8 @@ github.com/ipfs/dir-index-html v1.0.3/go.mod h1:TG9zbaH/+4MnkGel0xF4SLNhk+YZvBNo github.com/ipfs/go-bitswap v0.0.3/go.mod h1:jadAZYsP/tcRMl47ZhFxhaNuDQoXawT8iHMg+iFoQbg= github.com/ipfs/go-bitswap v0.0.7 h1:BJwx7Kh5W845l10bOckkAJiNrT6XXWNaE8neK7H57q4= github.com/ipfs/go-bitswap v0.0.7/go.mod h1:++LZRc+e1/ZxYsZq7QLKIPQvybh+70TMgeGuX+WB0pY= +github.com/ipfs/go-bitswap v0.0.8-0.20190704155249-cbb485998356 h1:DNc/6p5YnnHWnKab5Lmg6BS4CsRjWyYvapaIsQELAJg= +github.com/ipfs/go-bitswap v0.0.8-0.20190704155249-cbb485998356/go.mod h1:kYh8QssUH3mAuCwV3WmfcSqwJLhQC4WynKpd1hxt85A= github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc= github.com/ipfs/go-block-format v0.0.2 h1:qPDvcP19izTjU8rgo6p7gTXZlkMkF5bz5G3fqIsSCPE= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= @@ -221,6 +228,8 @@ github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46U github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8= github.com/ipfs/go-ds-badger v0.0.3 h1:sVYE2YlCzltznTZeAP1S+bp3qipz7VzogfZDtf6tGq0= github.com/ipfs/go-ds-badger v0.0.3/go.mod h1:7AzMKCsGav0u46HpdLiAEAOqizR1H6AZsjpHpQSPYCQ= +github.com/ipfs/go-ds-badger v0.0.5 h1:dxKuqw5T1Jm8OuV+lchA76H9QZFyPKZeLuT6bN42hJQ= +github.com/ipfs/go-ds-badger v0.0.5/go.mod h1:g5AuuCGmr7efyzQhLL8MzwqcauPojGPUaHzfGTzuE3s= github.com/ipfs/go-ds-flatfs v0.0.2 h1:1zujtU5bPBH6B8roE+TknKIbBCrpau865xUk0dH3x2A= github.com/ipfs/go-ds-flatfs v0.0.2/go.mod h1:YsMGWjUieue+smePAWeH/YhHtlmEMnEGhiwIn6K6rEM= github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= @@ -232,6 +241,8 @@ github.com/ipfs/go-fs-lock v0.0.1 h1:XHX8uW4jQBYWHj59XXcjg7BHlHxV9ZOYs6Y43yb7/l0 github.com/ipfs/go-fs-lock v0.0.1/go.mod h1:DNBekbboPKcxs1aukPSaOtFA3QfSdi5C855v0i9XJ8Y= github.com/ipfs/go-ipfs v0.4.21 h1:KB4k3U90cesx60MwHEOqUoSCquZ+JXXHNdw0HIKBusc= github.com/ipfs/go-ipfs v0.4.21/go.mod h1:T9zAmGO+rzbvLjDUm0DHtYXtdT4GhWOZeCPztgmt2V8= +github.com/ipfs/go-ipfs v0.4.22 h1:dDUSkIj0kSrMJW55wOm9P+mYewxCDSATy+PGZrShdfI= +github.com/ipfs/go-ipfs v0.4.22/go.mod h1:vgIn+MMrMEN2Yd6AWxAZVP/+YGI8dSGSgwiTEU6R2kw= github.com/ipfs/go-ipfs-addr v0.0.1 h1:DpDFybnho9v3/a1dzJ5KnWdThWD1HrFLpQ+tWIyBaFI= github.com/ipfs/go-ipfs-addr v0.0.1/go.mod h1:uKTDljHT3Q3SUWzDLp3aYUi8MrY32fgNgogsIa0npjg= github.com/ipfs/go-ipfs-blockstore v0.0.1 h1:O9n3PbmTYZoNhkgkEyrXTznbmktIXif62xLX+8dPHzc= @@ -292,6 +303,8 @@ github.com/ipfs/go-path v0.0.4 h1:zG/id80tV51XAfvCsRJIEGQSHGuTDBi8RWrtr3EfcfY= github.com/ipfs/go-path v0.0.4/go.mod h1:zIRQUez3LuQIU25zFjC2hpBTHimWx7VK5bjZgRLbbdo= github.com/ipfs/go-peertaskqueue v0.0.4 h1:i0JprfjjILYcWM1xguO/1MCS8XKVxLSl+ECEVr6i8nw= github.com/ipfs/go-peertaskqueue v0.0.4/go.mod h1:03H8fhyeMfKNFWqzYEVyMbcPUeYrqP1MX6Kd+aN+rMQ= +github.com/ipfs/go-peertaskqueue v0.0.5-0.20190704154349-f09820a0a5b6 h1:/8zfOPbZ8q0YadOu7snck13TtYW9f08v65XDWag9+jU= +github.com/ipfs/go-peertaskqueue v0.0.5-0.20190704154349-f09820a0a5b6/go.mod h1:03H8fhyeMfKNFWqzYEVyMbcPUeYrqP1MX6Kd+aN+rMQ= github.com/ipfs/go-todocounter v0.0.1 h1:kITWA5ZcQZfrUnDNkRn04Xzh0YFaDFXsoO2A81Eb6Lw= github.com/ipfs/go-todocounter v0.0.1/go.mod h1:l5aErvQc8qKE2r7NDMjmq5UNAvuZy0rC8BHOplkWvZ4= github.com/ipfs/go-unixfs v0.0.4/go.mod h1:eIo/p9ADu/MFOuyxzwU+Th8D6xoxU//r590vUpWyfz8= @@ -571,6 +584,8 @@ github.com/multiformats/go-multiaddr v0.0.4/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lg github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2 h1:/Bbsgsy3R6e3jf2qBahzNHzww6usYaZ0NhNH3sqdFS8= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= +github.com/multiformats/go-multiaddr-dns v0.0.3 h1:P19q/k9jwmtgh+qXFkKfgFM7rCg/9l5AVqh7VNxSXhs= +github.com/multiformats/go-multiaddr-dns v0.0.3/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-net v0.0.1 h1:76O59E3FavvHqNg7jvzWzsPSW5JSi/ek0E4eiDVbg9g= github.com/multiformats/go-multiaddr-net v0.0.1/go.mod h1:nw6HSxNmCIQH27XPGBuX+d1tnvM7ihcFwHMSstNAVUU= github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmrJR+ubhT9qA= @@ -711,6 +726,8 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.4 h1:S0tLZ3VOKl2Te0hpq8+ke0eSJPfCnNTPiDlsfwi1/NE= github.com/spf13/cobra v0.0.4/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -866,6 +883,8 @@ golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5 h1:f005F/Jl5JLP036x7QIvUVhNT golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae h1:xiXzMMEQdQcric9hXtr1QU98MHunKK7OTtsoU6bYWs4= golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/lib/lib.go b/lib/lib.go index 459978793..6c9084712 100644 --- a/lib/lib.go +++ b/lib/lib.go @@ -573,6 +573,27 @@ type Instance struct { rpc *rpc.Client } +// Connect takes an instance online +func (inst *Instance) Connect(ctx context.Context) (err error) { + if err = inst.node.GoOnline(); err != nil { + log.Debugf("taking node online: %s", err.Error()) + return + } + + // for now if we have an IPFS node instance, node.GoOnline has to make a new + // instance to connect properly. If remoteClient retains the reference to the + // old instance, we run into issues where the online instance can't "see" + // the additions. We fix that by re-initializing the client with the new + // instance + if inst.remoteClient, err = remote.NewClient(inst.node); err != nil { + log.Debugf("initializing remote client: %s", err.Error()) + return + } + + // chriswhong/usgs_earthquakes + return nil +} + // Context returns the base context for this instance func (inst *Instance) Context() context.Context { return inst.ctx diff --git a/p2p/node.go b/p2p/node.go index a1b499311..b83cc4761 100644 --- a/p2p/node.go +++ b/p2p/node.go @@ -252,6 +252,13 @@ func (n *QriNode) ipfsNode() (*core.IpfsNode, error) { return nil, fmt.Errorf("not using IPFS") } +// IPFS exposes the core.IPFS node if one exists. +// This is currently required by things like remoteClient in other packages, +// which don't work properly with the CoreAPI implementation +func (n *QriNode) IPFS() (*core.IpfsNode, error) { + return n.ipfsNode() +} + // GetIPFSNamesys returns a namesystem from IPFS func (n *QriNode) GetIPFSNamesys() (namesys.NameSystem, error) { ipfsn, err := n.ipfsNode() diff --git a/remote/client.go b/remote/client.go index 83f46b89c..59eee1f51 100644 --- a/remote/client.go +++ b/remote/client.go @@ -11,6 +11,7 @@ import ( "strings" "time" + coreiface "github.com/ipfs/interface-go-ipfs-core" crypto "github.com/libp2p/go-libp2p-crypto" peer "github.com/libp2p/go-libp2p-peer" "github.com/multiformats/go-multihash" @@ -42,8 +43,9 @@ func Address(cfg *config.Config, name string) (addr string, err error) { // Client issues requests to a remote type Client struct { - pk crypto.PrivKey - ds *dsync.Dsync + pk crypto.PrivKey + ds *dsync.Dsync + capi coreiface.CoreAPI } // NewClient creates a client @@ -67,11 +69,19 @@ func NewClient(node *p2p.QriNode) (*Client, error) { }) return &Client{ - pk: node.Repo.PrivateKey(), - ds: ds, + pk: node.Repo.PrivateKey(), + ds: ds, + capi: capi, }, nil } +// CoreAPI exposes this client's CoreApi +// TODO (b5) - this shouldn't be necessary, currently being exposed to debug +// Adding a dataset in actions.AddDataset +func (c *Client) CoreAPI() coreiface.CoreAPI { + return c.capi +} + // PushDataset pushes the contents of a dataset to a remote func (c *Client) PushDataset(ctx context.Context, ref repo.DatasetRef, remoteAddr string) error { if c == nil {