fix(add): need to re-initialize add when connecting

if we have an IPFS node instance, node.GoOnline has to make a new
instance to connect properly. If remoteClient retains the reference to the
old instance, we run into issues where the online instance can't "see"
the additions. We fix that by re-initializing the client with the new
instance.

Debugging this one was not fun.

actions/dataset.go

@@ -151,6 +151,8 @@ func AddDataset(ctx context.Context, node *p2p.QriNode, rc *remote.Client, remot
  Error error
  }
 
+ fetchCtx, cancelFetch := context.WithCancel(ctx)
+ defer cancelFetch()
  responses := make(chan addResponse)
  tasks := 0
 
@@ -172,13 +174,13 @@ func AddDataset(ctx context.Context, node *p2p.QriNode, rc *remote.Client, remot
  responses <- res
  }()
 
- if err := rc.PullDataset(ctx, ref, remoteAddr); err != nil {
+ if err := rc.PullDataset(fetchCtx, ref, remoteAddr); err != nil {
  res.Error = err
  return
  }
  node.LocalStreams.PrintErr("🗼 fetched from registry\n")
  if pinner, ok := node.Repo.Store().(cafs.Pinner); ok {
- err := pinner.Pin(ctx, ref.Path, true)
+ err := pinner.Pin(fetchCtx, ref.Path, true)
  res.Error = err
  }
  }(refCopy)
@@ -187,7 +189,7 @@ func AddDataset(ctx context.Context, node *p2p.QriNode, rc *remote.Client, remot
  if node.Online {
  tasks++
  go func() {
- err := base.FetchDataset(ctx, node.Repo, ref, true, true)
+ err := base.FetchDataset(fetchCtx, node.Repo, ref, true, true)
  responses <- addResponse{
  Ref: ref,
  Error: err,
@@ -204,6 +206,7 @@ func AddDataset(ctx context.Context, node *p2p.QriNode, rc *remote.Client, remot
  res := <-responses
  err = res.Error
  if err == nil {
+ cancelFetch()
  success = true
  *ref = *res.Ref
  break

api/api.go

@@ -49,9 +49,8 @@ func (s Server) Serve(ctx context.Context) (err error) {
  node := s.Node()
  cfg := s.Config()
 
- if err = node.GoOnline(); err != nil {
- fmt.Println("serving error", cfg.P2P.Enabled)
- return
+ if err := s.Instance.Connect(ctx); err != nil {
+ return err
  }
 
  server := &http.Server{}

base/dataset.go

@@ -44,10 +44,19 @@ func OpenDataset(ctx context.Context, fsys qfs.Filesystem, ds *dataset.Dataset)
  return
  }
  }
+
+ // TODO (b5) - this is an error sometimes caused by failing to properly pin the
+ // rendered file. this'll really trip up if we're online, b/c ipfs will hang
+ // forever looking for the missing render dag :()
  if ds.Viz != nil && ds.Viz.RenderedFile() == nil {
  if err = ds.Viz.OpenRenderedFile(ctx, fsys); err != nil {
- log.Debug(err)
- return
+ if strings.Contains(err.Error(), "not found") {
+ log.Debug("skipping not-found viz script")
+ err = nil
+ } else {
+ log.Debug(err)
+ return
+ }
  }
  }
  return

go.mod

@@ -56,11 +56,10 @@ require (
  github.com/qri-io/varName v0.1.0
  github.com/sergi/go-diff v1.0.0
  github.com/sirupsen/logrus v1.2.0
- github.com/spf13/cobra v0.0.4
+ github.com/spf13/cobra v0.0.5
  github.com/theckman/go-flock v0.7.1
  go.starlark.net v0.0.0-20190528202925-30ae18b8564f
  golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
- golang.org/x/net v0.0.0-20190620200207-3b0461eec859 // indirect
- golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae
+ golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb
  gopkg.in/yaml.v2 v2.2.2
 )

go.sum

@@ -5,6 +5,8 @@ github.com/360EntSecGroup-Skylar/excelize v1.4.1 h1:l55mJb6rkkaUzOpSsgEeKYtS6/0g
 github.com/360EntSecGroup-Skylar/excelize v1.4.1/go.mod h1:vnax29X2usfl7HHkBrX5EvSCJcmH3dT9luvxzu8iGAE=
 github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7 h1:PqzgE6kAMi81xWQA2QIVxjWkFHptGgC547vchpUbtFo=
 github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
+github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkByhMAwYaFAX9w2l7vxvBQ5NMoxDrkhqhtn4=
+github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y=
 github.com/Kubuxu/gocovmerge v0.0.0-20161216165753-7ecaa51963cd/go.mod h1:bqoB8kInrTeEtYAwaIXoSRqdwnjQmFhsfusnzyui6yY=
@@ -82,10 +84,13 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018 h1:6xT9KW8zLC5IlbaIF5Q7JNieBoACT7iW0YTxQHR0in0=
 github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4=
 github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ=
+github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
 github.com/dgraph-io/badger v2.0.0-rc.2+incompatible h1:7KPp6xv5+wymkVUbkAnZZXvmDrJlf09m/7u1HG5lAYA=
 github.com/dgraph-io/badger v2.0.0-rc.2+incompatible/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ=
 github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f h1:dDxpBYafY/GYpcl+LS4Bn3ziLPuEdGRkRjYAbSlWxSA=
 github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
+github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
+github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
@@ -202,6 +207,8 @@ github.com/ipfs/dir-index-html v1.0.3/go.mod h1:TG9zbaH/+4MnkGel0xF4SLNhk+YZvBNo
 github.com/ipfs/go-bitswap v0.0.3/go.mod h1:jadAZYsP/tcRMl47ZhFxhaNuDQoXawT8iHMg+iFoQbg=
 github.com/ipfs/go-bitswap v0.0.7 h1:BJwx7Kh5W845l10bOckkAJiNrT6XXWNaE8neK7H57q4=
 github.com/ipfs/go-bitswap v0.0.7/go.mod h1:++LZRc+e1/ZxYsZq7QLKIPQvybh+70TMgeGuX+WB0pY=
+github.com/ipfs/go-bitswap v0.0.8-0.20190704155249-cbb485998356 h1:DNc/6p5YnnHWnKab5Lmg6BS4CsRjWyYvapaIsQELAJg=
+github.com/ipfs/go-bitswap v0.0.8-0.20190704155249-cbb485998356/go.mod h1:kYh8QssUH3mAuCwV3WmfcSqwJLhQC4WynKpd1hxt85A=
 github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc=
 github.com/ipfs/go-block-format v0.0.2 h1:qPDvcP19izTjU8rgo6p7gTXZlkMkF5bz5G3fqIsSCPE=
 github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
@@ -221,6 +228,8 @@ github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46U
 github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8=
 github.com/ipfs/go-ds-badger v0.0.3 h1:sVYE2YlCzltznTZeAP1S+bp3qipz7VzogfZDtf6tGq0=
 github.com/ipfs/go-ds-badger v0.0.3/go.mod h1:7AzMKCsGav0u46HpdLiAEAOqizR1H6AZsjpHpQSPYCQ=
+github.com/ipfs/go-ds-badger v0.0.5 h1:dxKuqw5T1Jm8OuV+lchA76H9QZFyPKZeLuT6bN42hJQ=
+github.com/ipfs/go-ds-badger v0.0.5/go.mod h1:g5AuuCGmr7efyzQhLL8MzwqcauPojGPUaHzfGTzuE3s=
 github.com/ipfs/go-ds-flatfs v0.0.2 h1:1zujtU5bPBH6B8roE+TknKIbBCrpau865xUk0dH3x2A=
 github.com/ipfs/go-ds-flatfs v0.0.2/go.mod h1:YsMGWjUieue+smePAWeH/YhHtlmEMnEGhiwIn6K6rEM=
 github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc=
@@ -232,6 +241,8 @@ github.com/ipfs/go-fs-lock v0.0.1 h1:XHX8uW4jQBYWHj59XXcjg7BHlHxV9ZOYs6Y43yb7/l0
 github.com/ipfs/go-fs-lock v0.0.1/go.mod h1:DNBekbboPKcxs1aukPSaOtFA3QfSdi5C855v0i9XJ8Y=
 github.com/ipfs/go-ipfs v0.4.21 h1:KB4k3U90cesx60MwHEOqUoSCquZ+JXXHNdw0HIKBusc=
 github.com/ipfs/go-ipfs v0.4.21/go.mod h1:T9zAmGO+rzbvLjDUm0DHtYXtdT4GhWOZeCPztgmt2V8=
+github.com/ipfs/go-ipfs v0.4.22 h1:dDUSkIj0kSrMJW55wOm9P+mYewxCDSATy+PGZrShdfI=
+github.com/ipfs/go-ipfs v0.4.22/go.mod h1:vgIn+MMrMEN2Yd6AWxAZVP/+YGI8dSGSgwiTEU6R2kw=
 github.com/ipfs/go-ipfs-addr v0.0.1 h1:DpDFybnho9v3/a1dzJ5KnWdThWD1HrFLpQ+tWIyBaFI=
 github.com/ipfs/go-ipfs-addr v0.0.1/go.mod h1:uKTDljHT3Q3SUWzDLp3aYUi8MrY32fgNgogsIa0npjg=
 github.com/ipfs/go-ipfs-blockstore v0.0.1 h1:O9n3PbmTYZoNhkgkEyrXTznbmktIXif62xLX+8dPHzc=
@@ -292,6 +303,8 @@ github.com/ipfs/go-path v0.0.4 h1:zG/id80tV51XAfvCsRJIEGQSHGuTDBi8RWrtr3EfcfY=
 github.com/ipfs/go-path v0.0.4/go.mod h1:zIRQUez3LuQIU25zFjC2hpBTHimWx7VK5bjZgRLbbdo=
 github.com/ipfs/go-peertaskqueue v0.0.4 h1:i0JprfjjILYcWM1xguO/1MCS8XKVxLSl+ECEVr6i8nw=
 github.com/ipfs/go-peertaskqueue v0.0.4/go.mod h1:03H8fhyeMfKNFWqzYEVyMbcPUeYrqP1MX6Kd+aN+rMQ=
+github.com/ipfs/go-peertaskqueue v0.0.5-0.20190704154349-f09820a0a5b6 h1:/8zfOPbZ8q0YadOu7snck13TtYW9f08v65XDWag9+jU=
+github.com/ipfs/go-peertaskqueue v0.0.5-0.20190704154349-f09820a0a5b6/go.mod h1:03H8fhyeMfKNFWqzYEVyMbcPUeYrqP1MX6Kd+aN+rMQ=
 github.com/ipfs/go-todocounter v0.0.1 h1:kITWA5ZcQZfrUnDNkRn04Xzh0YFaDFXsoO2A81Eb6Lw=
 github.com/ipfs/go-todocounter v0.0.1/go.mod h1:l5aErvQc8qKE2r7NDMjmq5UNAvuZy0rC8BHOplkWvZ4=
 github.com/ipfs/go-unixfs v0.0.4/go.mod h1:eIo/p9ADu/MFOuyxzwU+Th8D6xoxU//r590vUpWyfz8=
@@ -571,6 +584,8 @@ github.com/multiformats/go-multiaddr v0.0.4/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lg
 github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
 github.com/multiformats/go-multiaddr-dns v0.0.2 h1:/Bbsgsy3R6e3jf2qBahzNHzww6usYaZ0NhNH3sqdFS8=
 github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
+github.com/multiformats/go-multiaddr-dns v0.0.3 h1:P19q/k9jwmtgh+qXFkKfgFM7rCg/9l5AVqh7VNxSXhs=
+github.com/multiformats/go-multiaddr-dns v0.0.3/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q=
 github.com/multiformats/go-multiaddr-net v0.0.1 h1:76O59E3FavvHqNg7jvzWzsPSW5JSi/ek0E4eiDVbg9g=
 github.com/multiformats/go-multiaddr-net v0.0.1/go.mod h1:nw6HSxNmCIQH27XPGBuX+d1tnvM7ihcFwHMSstNAVUU=
 github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmrJR+ubhT9qA=
@@ -711,6 +726,8 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU
 github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v0.0.4 h1:S0tLZ3VOKl2Te0hpq8+ke0eSJPfCnNTPiDlsfwi1/NE=
 github.com/spf13/cobra v0.0.4/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
+github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
+github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
 github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
@@ -866,6 +883,8 @@ golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5 h1:f005F/Jl5JLP036x7QIvUVhNT
 golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae h1:xiXzMMEQdQcric9hXtr1QU98MHunKK7OTtsoU6bYWs4=
 golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k=
+golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

lib/lib.go

@@ -573,6 +573,27 @@ type Instance struct {
  rpc *rpc.Client
 }
 
+// Connect takes an instance online
+func (inst *Instance) Connect(ctx context.Context) (err error) {
+ if err = inst.node.GoOnline(); err != nil {
+ log.Debugf("taking node online: %s", err.Error())
+ return
+ }
+
+ // for now if we have an IPFS node instance, node.GoOnline has to make a new
+ // instance to connect properly. If remoteClient retains the reference to the
+ // old instance, we run into issues where the online instance can't "see"
+ // the additions. We fix that by re-initializing the client with the new
+ // instance
+ if inst.remoteClient, err = remote.NewClient(inst.node); err != nil {
+ log.Debugf("initializing remote client: %s", err.Error())
+ return
+ }
+
+ // chriswhong/usgs_earthquakes
+ return nil
+}
+
 // Context returns the base context for this instance
 func (inst *Instance) Context() context.Context {
  return inst.ctx

p2p/node.go

@@ -252,6 +252,13 @@ func (n *QriNode) ipfsNode() (*core.IpfsNode, error) {
  return nil, fmt.Errorf("not using IPFS")
 }
 
+// IPFS exposes the core.IPFS node if one exists.
+// This is currently required by things like remoteClient in other packages,
+// which don't work properly with the CoreAPI implementation
+func (n *QriNode) IPFS() (*core.IpfsNode, error) {
+ return n.ipfsNode()
+}
+
 // GetIPFSNamesys returns a namesystem from IPFS
 func (n *QriNode) GetIPFSNamesys() (namesys.NameSystem, error) {
  ipfsn, err := n.ipfsNode()

remote/client.go

@@ -11,6 +11,7 @@ import (
  "strings"
  "time"
 
+ coreiface "github.com/ipfs/interface-go-ipfs-core"
  crypto "github.com/libp2p/go-libp2p-crypto"
  peer "github.com/libp2p/go-libp2p-peer"
  "github.com/multiformats/go-multihash"
@@ -42,8 +43,9 @@ func Address(cfg *config.Config, name string) (addr string, err error) {
 
 // Client issues requests to a remote
 type Client struct {
- pk crypto.PrivKey
- ds *dsync.Dsync
+ pk crypto.PrivKey
+ ds *dsync.Dsync
+ capi coreiface.CoreAPI
 }
 
 // NewClient creates a client
@@ -67,11 +69,19 @@ func NewClient(node *p2p.QriNode) (*Client, error) {
  })
 
  return &Client{
- pk: node.Repo.PrivateKey(),
- ds: ds,
+ pk: node.Repo.PrivateKey(),
+ ds: ds,
+ capi: capi,
  }, nil
 }
 
+// CoreAPI exposes this client's CoreApi
+// TODO (b5) - this shouldn't be necessary, currently being exposed to debug
+// Adding a dataset in actions.AddDataset
+func (c *Client) CoreAPI() coreiface.CoreAPI {
+ return c.capi
+}
+
 // PushDataset pushes the contents of a dataset to a remote
 func (c *Client) PushDataset(ctx context.Context, ref repo.DatasetRef, remoteAddr string) error {
  if c == nil {