diff --git a/actions/dataset.go b/actions/dataset.go index 7dc8e9b13..97516af99 100644 --- a/actions/dataset.go +++ b/actions/dataset.go @@ -151,6 +151,8 @@ func AddDataset(ctx context.Context, node *p2p.QriNode, rc *remote.Client, remot Error error } + fetchCtx, cancelFetch := context.WithCancel(ctx) + defer cancelFetch() responses := make(chan addResponse) tasks := 0 @@ -172,13 +174,13 @@ func AddDataset(ctx context.Context, node *p2p.QriNode, rc *remote.Client, remot responses <- res }() - if err := rc.PullDataset(ctx, ref, remoteAddr); err != nil { + if err := rc.PullDataset(fetchCtx, ref, remoteAddr); err != nil { res.Error = err return } node.LocalStreams.PrintErr("🗼 fetched from registry\n") if pinner, ok := node.Repo.Store().(cafs.Pinner); ok { - err := pinner.Pin(ctx, ref.Path, true) + err := pinner.Pin(fetchCtx, ref.Path, true) res.Error = err } }(refCopy) @@ -187,7 +189,7 @@ func AddDataset(ctx context.Context, node *p2p.QriNode, rc *remote.Client, remot if node.Online { tasks++ go func() { - err := base.FetchDataset(ctx, node.Repo, ref, true, true) + err := base.FetchDataset(fetchCtx, node.Repo, ref, true, true) responses <- addResponse{ Ref: ref, Error: err, @@ -204,6 +206,7 @@ func AddDataset(ctx context.Context, node *p2p.QriNode, rc *remote.Client, remot res := <-responses err = res.Error if err == nil { + cancelFetch() success = true *ref = *res.Ref break diff --git a/api/api.go b/api/api.go index d2eb1c394..87c6b79ef 100644 --- a/api/api.go +++ b/api/api.go @@ -49,9 +49,8 @@ func (s Server) Serve(ctx context.Context) (err error) { node := s.Node() cfg := s.Config() - if err = node.GoOnline(); err != nil { - fmt.Println("serving error", cfg.P2P.Enabled) - return + if err := s.Instance.Connect(ctx); err != nil { + return err } server := &http.Server{} diff --git a/base/dataset.go b/base/dataset.go index d921efae1..55c1d2fc1 100644 --- a/base/dataset.go +++ b/base/dataset.go @@ -44,10 +44,19 @@ func OpenDataset(ctx context.Context, fsys qfs.Filesystem, ds *dataset.Dataset) return } } + + // TODO (b5) - this is an error sometimes caused by failing to properly pin the + // rendered file. this'll really trip up if we're online, b/c ipfs will hang + // forever looking for the missing render dag :() if ds.Viz != nil && ds.Viz.RenderedFile() == nil { if err = ds.Viz.OpenRenderedFile(ctx, fsys); err != nil { - log.Debug(err) - return + if strings.Contains(err.Error(), "not found") { + log.Debug("skipping not-found viz script") + err = nil + } else { + log.Debug(err) + return + } } } return diff --git a/go.mod b/go.mod index 522d3c014..222ae9bb2 100644 --- a/go.mod +++ b/go.mod @@ -56,11 +56,10 @@ require ( github.com/qri-io/varName v0.1.0 github.com/sergi/go-diff v1.0.0 github.com/sirupsen/logrus v1.2.0 - github.com/spf13/cobra v0.0.4 + github.com/spf13/cobra v0.0.5 github.com/theckman/go-flock v0.7.1 go.starlark.net v0.0.0-20190528202925-30ae18b8564f golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 - golang.org/x/net v0.0.0-20190620200207-3b0461eec859 // indirect - golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae + golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb gopkg.in/yaml.v2 v2.2.2 ) diff --git a/go.sum b/go.sum index ffb5ba030..153a87e3d 100644 --- a/go.sum +++ b/go.sum @@ -5,6 +5,8 @@ github.com/360EntSecGroup-Skylar/excelize v1.4.1 h1:l55mJb6rkkaUzOpSsgEeKYtS6/0g github.com/360EntSecGroup-Skylar/excelize v1.4.1/go.mod h1:vnax29X2usfl7HHkBrX5EvSCJcmH3dT9luvxzu8iGAE= github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7 h1:PqzgE6kAMi81xWQA2QIVxjWkFHptGgC547vchpUbtFo= github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkByhMAwYaFAX9w2l7vxvBQ5NMoxDrkhqhtn4= +github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/Kubuxu/gocovmerge v0.0.0-20161216165753-7ecaa51963cd/go.mod h1:bqoB8kInrTeEtYAwaIXoSRqdwnjQmFhsfusnzyui6yY= @@ -82,10 +84,13 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018 h1:6xT9KW8zLC5IlbaIF5Q7JNieBoACT7iW0YTxQHR0in0= github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4= github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= +github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger v2.0.0-rc.2+incompatible h1:7KPp6xv5+wymkVUbkAnZZXvmDrJlf09m/7u1HG5lAYA= github.com/dgraph-io/badger v2.0.0-rc.2+incompatible/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f h1:dDxpBYafY/GYpcl+LS4Bn3ziLPuEdGRkRjYAbSlWxSA= github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -202,6 +207,8 @@ github.com/ipfs/dir-index-html v1.0.3/go.mod h1:TG9zbaH/+4MnkGel0xF4SLNhk+YZvBNo github.com/ipfs/go-bitswap v0.0.3/go.mod h1:jadAZYsP/tcRMl47ZhFxhaNuDQoXawT8iHMg+iFoQbg= github.com/ipfs/go-bitswap v0.0.7 h1:BJwx7Kh5W845l10bOckkAJiNrT6XXWNaE8neK7H57q4= github.com/ipfs/go-bitswap v0.0.7/go.mod h1:++LZRc+e1/ZxYsZq7QLKIPQvybh+70TMgeGuX+WB0pY= +github.com/ipfs/go-bitswap v0.0.8-0.20190704155249-cbb485998356 h1:DNc/6p5YnnHWnKab5Lmg6BS4CsRjWyYvapaIsQELAJg= +github.com/ipfs/go-bitswap v0.0.8-0.20190704155249-cbb485998356/go.mod h1:kYh8QssUH3mAuCwV3WmfcSqwJLhQC4WynKpd1hxt85A= github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc= github.com/ipfs/go-block-format v0.0.2 h1:qPDvcP19izTjU8rgo6p7gTXZlkMkF5bz5G3fqIsSCPE= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= @@ -221,6 +228,8 @@ github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46U github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8= github.com/ipfs/go-ds-badger v0.0.3 h1:sVYE2YlCzltznTZeAP1S+bp3qipz7VzogfZDtf6tGq0= github.com/ipfs/go-ds-badger v0.0.3/go.mod h1:7AzMKCsGav0u46HpdLiAEAOqizR1H6AZsjpHpQSPYCQ= +github.com/ipfs/go-ds-badger v0.0.5 h1:dxKuqw5T1Jm8OuV+lchA76H9QZFyPKZeLuT6bN42hJQ= +github.com/ipfs/go-ds-badger v0.0.5/go.mod h1:g5AuuCGmr7efyzQhLL8MzwqcauPojGPUaHzfGTzuE3s= github.com/ipfs/go-ds-flatfs v0.0.2 h1:1zujtU5bPBH6B8roE+TknKIbBCrpau865xUk0dH3x2A= github.com/ipfs/go-ds-flatfs v0.0.2/go.mod h1:YsMGWjUieue+smePAWeH/YhHtlmEMnEGhiwIn6K6rEM= github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= @@ -232,6 +241,8 @@ github.com/ipfs/go-fs-lock v0.0.1 h1:XHX8uW4jQBYWHj59XXcjg7BHlHxV9ZOYs6Y43yb7/l0 github.com/ipfs/go-fs-lock v0.0.1/go.mod h1:DNBekbboPKcxs1aukPSaOtFA3QfSdi5C855v0i9XJ8Y= github.com/ipfs/go-ipfs v0.4.21 h1:KB4k3U90cesx60MwHEOqUoSCquZ+JXXHNdw0HIKBusc= github.com/ipfs/go-ipfs v0.4.21/go.mod h1:T9zAmGO+rzbvLjDUm0DHtYXtdT4GhWOZeCPztgmt2V8= +github.com/ipfs/go-ipfs v0.4.22 h1:dDUSkIj0kSrMJW55wOm9P+mYewxCDSATy+PGZrShdfI= +github.com/ipfs/go-ipfs v0.4.22/go.mod h1:vgIn+MMrMEN2Yd6AWxAZVP/+YGI8dSGSgwiTEU6R2kw= github.com/ipfs/go-ipfs-addr v0.0.1 h1:DpDFybnho9v3/a1dzJ5KnWdThWD1HrFLpQ+tWIyBaFI= github.com/ipfs/go-ipfs-addr v0.0.1/go.mod h1:uKTDljHT3Q3SUWzDLp3aYUi8MrY32fgNgogsIa0npjg= github.com/ipfs/go-ipfs-blockstore v0.0.1 h1:O9n3PbmTYZoNhkgkEyrXTznbmktIXif62xLX+8dPHzc= @@ -292,6 +303,8 @@ github.com/ipfs/go-path v0.0.4 h1:zG/id80tV51XAfvCsRJIEGQSHGuTDBi8RWrtr3EfcfY= github.com/ipfs/go-path v0.0.4/go.mod h1:zIRQUez3LuQIU25zFjC2hpBTHimWx7VK5bjZgRLbbdo= github.com/ipfs/go-peertaskqueue v0.0.4 h1:i0JprfjjILYcWM1xguO/1MCS8XKVxLSl+ECEVr6i8nw= github.com/ipfs/go-peertaskqueue v0.0.4/go.mod h1:03H8fhyeMfKNFWqzYEVyMbcPUeYrqP1MX6Kd+aN+rMQ= +github.com/ipfs/go-peertaskqueue v0.0.5-0.20190704154349-f09820a0a5b6 h1:/8zfOPbZ8q0YadOu7snck13TtYW9f08v65XDWag9+jU= +github.com/ipfs/go-peertaskqueue v0.0.5-0.20190704154349-f09820a0a5b6/go.mod h1:03H8fhyeMfKNFWqzYEVyMbcPUeYrqP1MX6Kd+aN+rMQ= github.com/ipfs/go-todocounter v0.0.1 h1:kITWA5ZcQZfrUnDNkRn04Xzh0YFaDFXsoO2A81Eb6Lw= github.com/ipfs/go-todocounter v0.0.1/go.mod h1:l5aErvQc8qKE2r7NDMjmq5UNAvuZy0rC8BHOplkWvZ4= github.com/ipfs/go-unixfs v0.0.4/go.mod h1:eIo/p9ADu/MFOuyxzwU+Th8D6xoxU//r590vUpWyfz8= @@ -571,6 +584,8 @@ github.com/multiformats/go-multiaddr v0.0.4/go.mod h1:xKVEak1K9cS1VdmPZW3LSIb6lg github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2 h1:/Bbsgsy3R6e3jf2qBahzNHzww6usYaZ0NhNH3sqdFS8= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= +github.com/multiformats/go-multiaddr-dns v0.0.3 h1:P19q/k9jwmtgh+qXFkKfgFM7rCg/9l5AVqh7VNxSXhs= +github.com/multiformats/go-multiaddr-dns v0.0.3/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-net v0.0.1 h1:76O59E3FavvHqNg7jvzWzsPSW5JSi/ek0E4eiDVbg9g= github.com/multiformats/go-multiaddr-net v0.0.1/go.mod h1:nw6HSxNmCIQH27XPGBuX+d1tnvM7ihcFwHMSstNAVUU= github.com/multiformats/go-multibase v0.0.1 h1:PN9/v21eLywrFWdFNsFKaU04kLJzuYzmrJR+ubhT9qA= @@ -711,6 +726,8 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.4 h1:S0tLZ3VOKl2Te0hpq8+ke0eSJPfCnNTPiDlsfwi1/NE= github.com/spf13/cobra v0.0.4/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -866,6 +883,8 @@ golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5 h1:f005F/Jl5JLP036x7QIvUVhNT golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae h1:xiXzMMEQdQcric9hXtr1QU98MHunKK7OTtsoU6bYWs4= golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/lib/lib.go b/lib/lib.go index 459978793..6c9084712 100644 --- a/lib/lib.go +++ b/lib/lib.go @@ -573,6 +573,27 @@ type Instance struct { rpc *rpc.Client } +// Connect takes an instance online +func (inst *Instance) Connect(ctx context.Context) (err error) { + if err = inst.node.GoOnline(); err != nil { + log.Debugf("taking node online: %s", err.Error()) + return + } + + // for now if we have an IPFS node instance, node.GoOnline has to make a new + // instance to connect properly. If remoteClient retains the reference to the + // old instance, we run into issues where the online instance can't "see" + // the additions. We fix that by re-initializing the client with the new + // instance + if inst.remoteClient, err = remote.NewClient(inst.node); err != nil { + log.Debugf("initializing remote client: %s", err.Error()) + return + } + + // chriswhong/usgs_earthquakes + return nil +} + // Context returns the base context for this instance func (inst *Instance) Context() context.Context { return inst.ctx diff --git a/p2p/node.go b/p2p/node.go index a1b499311..b83cc4761 100644 --- a/p2p/node.go +++ b/p2p/node.go @@ -252,6 +252,13 @@ func (n *QriNode) ipfsNode() (*core.IpfsNode, error) { return nil, fmt.Errorf("not using IPFS") } +// IPFS exposes the core.IPFS node if one exists. +// This is currently required by things like remoteClient in other packages, +// which don't work properly with the CoreAPI implementation +func (n *QriNode) IPFS() (*core.IpfsNode, error) { + return n.ipfsNode() +} + // GetIPFSNamesys returns a namesystem from IPFS func (n *QriNode) GetIPFSNamesys() (namesys.NameSystem, error) { ipfsn, err := n.ipfsNode() diff --git a/remote/client.go b/remote/client.go index 83f46b89c..59eee1f51 100644 --- a/remote/client.go +++ b/remote/client.go @@ -11,6 +11,7 @@ import ( "strings" "time" + coreiface "github.com/ipfs/interface-go-ipfs-core" crypto "github.com/libp2p/go-libp2p-crypto" peer "github.com/libp2p/go-libp2p-peer" "github.com/multiformats/go-multihash" @@ -42,8 +43,9 @@ func Address(cfg *config.Config, name string) (addr string, err error) { // Client issues requests to a remote type Client struct { - pk crypto.PrivKey - ds *dsync.Dsync + pk crypto.PrivKey + ds *dsync.Dsync + capi coreiface.CoreAPI } // NewClient creates a client @@ -67,11 +69,19 @@ func NewClient(node *p2p.QriNode) (*Client, error) { }) return &Client{ - pk: node.Repo.PrivateKey(), - ds: ds, + pk: node.Repo.PrivateKey(), + ds: ds, + capi: capi, }, nil } +// CoreAPI exposes this client's CoreApi +// TODO (b5) - this shouldn't be necessary, currently being exposed to debug +// Adding a dataset in actions.AddDataset +func (c *Client) CoreAPI() coreiface.CoreAPI { + return c.capi +} + // PushDataset pushes the contents of a dataset to a remote func (c *Client) PushDataset(ctx context.Context, ref repo.DatasetRef, remoteAddr string) error { if c == nil {