From 79371c02c7c971e357d79192f15ff360db6d4f6d Mon Sep 17 00:00:00 2001 From: b5 Date: Thu, 14 Dec 2017 08:58:14 -0500 Subject: [PATCH] feat(cmd.Init): brand new initialization process lotsa fixes to the initialization process. I'd do a proper writeup but alas no time. --- api/config.go | 3 + api/server.go | 8 ++- cmd/config.go | 37 ++++++----- cmd/init.go | 179 ++++++++++++++++++++++++++++++++++++++++++++++++-- cmd/repo.go | 5 +- cmd/server.go | 60 +++++++++-------- 6 files changed, 236 insertions(+), 56 deletions(-) diff --git a/api/config.go b/api/config.go index b66e07b2c..d10369ef2 100644 --- a/api/config.go +++ b/api/config.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/qri-io/qri/logging" + "github.com/qri-io/qri/p2p" ) // server modes @@ -65,6 +66,8 @@ type Config struct { Online bool // list of addresses to bootsrap qri peers on BoostrapAddrs []string + // PostP2POnlineHook is a chance to call a function after starting P2P services + PostP2POnlineHook func(*p2p.QriNode) } // Validate returns nil if this configuration is valid, diff --git a/api/server.go b/api/server.go index cef80e4fb..5ce64c9cf 100644 --- a/api/server.go +++ b/api/server.go @@ -63,7 +63,13 @@ func New(r repo.Repo, options ...func(*Config)) (s *Server, err error) { s.log.Info("running qri in offline mode, no peer-2-peer connections") } - s.qriNode.StartOnlineServices() + err = s.qriNode.StartOnlineServices() + if err != nil { + return nil, fmt.Errorf("error starting P2P service: %s", err.Error()) + } + if cfg.PostP2POnlineHook != nil { + cfg.PostP2POnlineHook(s.qriNode) + } // p2p.PrintSwarmAddrs(qriNode) return s, nil } diff --git a/cmd/config.go b/cmd/config.go index 538e589fb..1ed658c42 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -5,20 +5,19 @@ import ( "os" "path/filepath" - "github.com/qri-io/qri/core" "github.com/qri-io/qri/p2p" "github.com/spf13/cobra" "github.com/spf13/viper" "gopkg.in/yaml.v2" ) -// Config configures the behaviour of qri +// Config configures the behavior of qri type Config struct { // Initialized is a flag for when this repo has been properly initialized at least once. // used to check weather default datasets should be added or not Initialized bool // Identity Configuration details - Identity IdentityCfg + // Identity IdentityCfg // List of nodes to boostrap to Bootstrap []string // Datastore configuration details @@ -28,14 +27,14 @@ type Config struct { } // IdentityCfg holds details about user identity & configuration -type IdentityCfg struct { - // ID to feed to IPFS node, and for profile identification - PeerID string - // PrivateKey for - PrivateKey string - // Profile - Profile *core.Profile -} +// type IdentityCfg struct { +// // ID to feed to IPFS node, and for profile identification +// PeerID string +// // PrivateKey for +// PrivateKey string +// // Profile +// Profile *core.Profile +// } // DatastoreCfg configures the underlying IPFS datastore. WIP. // type DatastoreCfg struct { @@ -92,13 +91,15 @@ func configFilepath() string { return path } -// func EnsureConfigFile() (bool, error) { -// if _, err := os.Stat(configFilepath()); os.IsNotExist(err) { -// fmt.Println("writing config file") -// return true, WriteConfigFile(defaultCfg) -// } -// return false, nil -// } +func ReadConfigFile() (*Config, error) { + data, err := ioutil.ReadFile(configFilepath()) + if err != nil { + return nil, err + } + cfg := Config{} + err = yaml.Unmarshal(data, &cfg) + return &cfg, err +} func WriteConfigFile(cfg *Config) error { data, err := yaml.Marshal(cfg) diff --git a/cmd/init.go b/cmd/init.go index fc73fbfcf..ad50f3313 100644 --- a/cmd/init.go +++ b/cmd/init.go @@ -1,7 +1,9 @@ package cmd import ( + "encoding/json" "fmt" + "github.com/qri-io/qri/core" "io/ioutil" "os" "path/filepath" @@ -10,12 +12,16 @@ import ( "github.com/spf13/cobra" "github.com/spf13/viper" "gopkg.in/yaml.v2" + + config "gx/ipfs/QmViBzgruNUoLNBnXcx8YWbDNwV8MNGEGKkLo6JGetygdw/go-ipfs/repo/config" ) var ( initOverwrite bool initIPFS bool initIPFSConfigFile string + initIdentityData string + initProfileData string ) // initCmd represents the init command @@ -27,7 +33,8 @@ var initCmd = &cobra.Command{ var cfgData []byte if QRIRepoInitialized() && !initOverwrite { - ErrExit(fmt.Errorf("repo already initialized. use --overwrite to overwrite this repo, erasing all data")) + // use --overwrite to overwrite this repo, erasing all data and deleting your account for good + ErrExit(fmt.Errorf("repo already initialized.")) } // if cfgFile is specified, override @@ -53,13 +60,57 @@ var initCmd = &cobra.Command{ err = viper.ReadInConfig() ExitIfErr(err) + if initIdentityData != "" { + err = readAtFile(&initIdentityData) + ExitIfErr(err) + + id := config.Identity{} + err = json.Unmarshal([]byte(initIdentityData), &id) + ExitIfErr(err) + + path := filepath.Join(os.TempDir(), "config") + data, err := json.Marshal(DefaaultIPFSConfig(id)) + ExitIfErr(err) + + err = ioutil.WriteFile(path, data, os.ModePerm) + ExitIfErr(err) + + initIPFSConfigFile = path + defer os.Remove(path) + } + if initIPFS { - err := ipfs.InitRepo(IpfsFsPath, initIPFSConfigFile) + err = ipfs.InitRepo(IpfsFsPath, initIPFSConfigFile) + ExitIfErr(err) + } + + if initProfileData != "" { + err = readAtFile(&initProfileData) + ExitIfErr(err) + + p := &core.Profile{} + err = json.Unmarshal([]byte(initProfileData), p) + ExitIfErr(err) + + pr, err := ProfileRequests(false) + ExitIfErr(err) + + res := &core.Profile{} + err = pr.SaveProfile(p, res) ExitIfErr(err) } }, } +func init() { + RootCmd.AddCommand(initCmd) + initCmd.Flags().BoolVarP(&initOverwrite, "overwrite", "", false, "overwrite repo if one exists") + initCmd.Flags().BoolVarP(&initIPFS, "init-ipfs", "", true, "initialize an IPFS repo if one isn't present") + // initCmd.Flags().StringVarP(&initIPFSConfigFile, "ipfs-config", "", "", "config file for initialization") + initCmd.Flags().StringVarP(&initIdentityData, "id", "", "", "json-encoded identity data, specify a filepath with '@' prefix") + initCmd.Flags().StringVarP(&initProfileData, "profile", "", "", "json-encoded user profile data, specify a filepath with '@' prefix") +} + // QRIRepoInitialized checks to see if a repository has been initialized at $QRI_PATH func QRIRepoInitialized() bool { // for now this just checks for an existing config file @@ -81,9 +132,123 @@ func initRepoIfEmpty(repoPath, configPath string) error { return nil } -func init() { - RootCmd.AddCommand(initCmd) - initCmd.Flags().BoolVarP(&initOverwrite, "overwrite", "", false, "overwrite repo if one exists") - initCmd.Flags().BoolVarP(&initIPFS, "init-ipfs", "", true, "initialize an IPFS repo if one isn't present") - initCmd.Flags().StringVarP(&initIPFSConfigFile, "ipfs-config", "", "", "config file for initialization") +// readAtFile is a unix curl inspired method. any data input that begins with "@" +// is assumed to instead be a filepath that should be read & replaced with the contents +// of the specified path +func readAtFile(data *string) error { + d := *data + if len(d) > 0 && d[0] == '@' { + fileData, err := ioutil.ReadFile(d[1:]) + if err != nil { + return err + } + *data = string(fileData) + } + return nil +} + +// TODO - this is a bit of a hack for the moment, will be removed later +func DefaaultIPFSConfig(identity config.Identity) *config.Config { + return &config.Config{ + Identity: identity, + Datastore: config.Datastore{ + StorageMax: "10GB", + StorageGCWatermark: 90, + GCPeriod: "1h", + Spec: map[string]interface{}{ + "mounts": []map[string]interface{}{ + { + "child": map[string]interface{}{ + "path": "blocks", + "shardFunc": "/repo/flatfs/shard/v1/next-to-last/2", + "sync": true, + "type": "flatfs", + }, + "mountpoint": "/blocks", + "prefix": "flatfs.datastore", + "type": "measure", + }, + { + "child": map[string]interface{}{ + "compression": "none", + "path": "datastore", + "type": "levelds", + }, + "mountpoint": "/", + "prefix": "leveldb.datastore", + "type": "measure", + }, + }, + "type": "mount", + }, + HashOnRead: false, + BloomFilterSize: 0, + }, + Bootstrap: []string{ + "/dnsaddr/bootstrap.libp2p.io/ipfs/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN", + "/dnsaddr/bootstrap.libp2p.io/ipfs/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa", + "/dnsaddr/bootstrap.libp2p.io/ipfs/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb", + "/dnsaddr/bootstrap.libp2p.io/ipfs/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt", + "/ip4/104.131.131.82/tcp/4001/ipfs/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", + "/ip4/104.236.179.241/tcp/4001/ipfs/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM", + "/ip4/128.199.219.111/tcp/4001/ipfs/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu", + "/ip4/104.236.76.40/tcp/4001/ipfs/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64", + "/ip4/178.62.158.247/tcp/4001/ipfs/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd", + "/ip6/2604:a880:1:20::203:d001/tcp/4001/ipfs/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM", + "/ip6/2400:6180:0:d0::151:6001/tcp/4001/ipfs/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu", + "/ip6/2604:a880:800:10::4a:5001/tcp/4001/ipfs/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64", + "/ip6/2a03:b0c0:0:1010::23:1001/tcp/4001/ipfs/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd", + }, + // setup the node's default addresses. + // NOTE: two swarm listen addrs, one tcp, one utp. + Addresses: config.Addresses{ + Swarm: []string{ + "/ip4/0.0.0.0/tcp/4001", + // "/ip4/0.0.0.0/udp/4002/utp", // disabled for now. + "/ip6/::/tcp/4001", + }, + Announce: []string{}, + NoAnnounce: []string{}, + API: "/ip4/127.0.0.1/tcp/5001", + Gateway: "/ip4/127.0.0.1/tcp/8080", + }, + + Discovery: config.Discovery{config.MDNS{ + Enabled: true, + Interval: 10, + }}, + + // setup the node mount points. + Mounts: config.Mounts{ + IPFS: "/ipfs", + IPNS: "/ipns", + }, + + Ipns: config.Ipns{ + ResolveCacheSize: 128, + }, + + Gateway: config.Gateway{ + RootRedirect: "", + Writable: false, + PathPrefixes: []string{}, + HTTPHeaders: map[string][]string{ + "Access-Control-Allow-Origin": []string{"*"}, + "Access-Control-Allow-Methods": []string{"GET"}, + "Access-Control-Allow-Headers": []string{"X-Requested-With", "Range"}, + }, + }, + Reprovider: config.Reprovider{ + Interval: "12h", + Strategy: "all", + }, + Swarm: config.SwarmConfig{ + ConnMgr: config.ConnMgr{ + LowWater: config.DefaultConnMgrLowWater, + HighWater: config.DefaultConnMgrHighWater, + GracePeriod: config.DefaultConnMgrGracePeriod.String(), + Type: "basic", + }, + }, + } } diff --git a/cmd/repo.go b/cmd/repo.go index 7e10b3f3f..83c553990 100644 --- a/cmd/repo.go +++ b/cmd/repo.go @@ -10,7 +10,6 @@ import ( "github.com/qri-io/qri/core" "github.com/qri-io/qri/repo" "github.com/qri-io/qri/repo/fs" - "github.com/spf13/viper" ) var r repo.Repo @@ -79,7 +78,7 @@ func SearchRequests(online bool) (*core.SearchRequests, error) { // RepoOrClient returns either a func RepoOrClient(online bool) (repo.Repo, *rpc.Client, error) { if fs, err := ipfs.NewFilestore(func(cfg *ipfs.StoreCfg) { - cfg.FsRepoPath = viper.GetString(IpfsFsPath) + cfg.FsRepoPath = IpfsFsPath cfg.Online = online }); err == nil { id := "" @@ -87,7 +86,7 @@ func RepoOrClient(online bool) (repo.Repo, *rpc.Client, error) { id = fs.Node().PeerHost.ID().Pretty() } - r, err := fs_repo.NewRepo(fs, viper.GetString(QriRepoPath), id) + r, err := fs_repo.NewRepo(fs, QriRepoPath, id) return r, nil, err } else if strings.Contains(err.Error(), "lock") { diff --git a/cmd/server.go b/cmd/server.go index f30ebc5b9..a88647ee5 100644 --- a/cmd/server.go +++ b/cmd/server.go @@ -1,9 +1,13 @@ package cmd import ( + "fmt" + "github.com/qri-io/qri/p2p" + "github.com/qri-io/analytics" "github.com/qri-io/cafs/memfs" "github.com/qri-io/qri/api" + "github.com/qri-io/qri/core" "github.com/qri-io/qri/repo" "github.com/qri-io/qri/repo/profile" "github.com/spf13/cobra" @@ -23,11 +27,6 @@ var serverCmd = &cobra.Command{ Short: "start a qri server", Long: ``, Run: func(cmd *cobra.Command, args []string) { - if serverInitIpfs { - err := initRepoIfEmpty(viper.GetString(IpfsFsPath), "") - ExitIfErr(err) - } - var ( r repo.Repo err error @@ -54,6 +53,7 @@ var serverCmd = &cobra.Command{ cfg.MemOnly = serverMemOnly cfg.Online = !serverOffline cfg.BoostrapAddrs = viper.GetStringSlice("bootstrap") + cfg.PostP2POnlineHook = addDefaultDatasets }) ExitIfErr(err) @@ -62,34 +62,40 @@ var serverCmd = &cobra.Command{ }, } -// Init sets up a repository with sensible defaults -func addDefaultDatasets() error { - // req, err := DatasetRequests(true) - // if err != nil { - // return err - // } +// Init sets up a repository with default datasets +func addDefaultDatasets(node *p2p.QriNode) { + cfg, err := ReadConfigFile() + if err != nil || cfg.Initialized { + return + } + + req := core.NewDatasetRequests(node.Repo, nil) + + for name, path := range cfg.DefaultDatasets { + fmt.Printf("attempting to add default dataset: %s\n", path) + res := &repo.DatasetRef{} + err := req.AddDataset(&core.AddParams{ + Hash: path, + Name: name, + }, res) + if err != nil { + fmt.Printf("add dataset %s error: %s\n", path, err.Error()) + return + } + fmt.Printf("added default dataset: %s\n", path) + } - // TODO - restore - // for name, ds := range defaultDatasets { - // fmt.Printf("attempting to add default dataset: %s\n", ds.String()) - // res := &repo.DatasetRef{} - // err := req.AddDataset(&core.AddParams{ - // Hash: ds.String(), - // Name: name, - // }, res) - // if err != nil { - // fmt.Printf("add dataset %s error: %s\n", ds.String(), err.Error()) - // return err - // } - // fmt.Printf("added default dataset: %s\n", ds.String()) - // } + cfg.Initialized = true + if err = WriteConfigFile(cfg); err != nil { + fmt.Printf("error writing config file: %s", err.Error()) + } - return nil + return } func init() { serverCmd.Flags().StringVarP(&serverCmdPort, "port", "p", api.DefaultPort, "port to start server on") - serverCmd.Flags().BoolVarP(&serverInitIpfs, "init-ipfs", "", false, "initialize a new default ipfs repo if empty") + // serverCmd.Flags().BoolVarP(&serverInitIpfs, "init-ipfs", "", false, "initialize a new default ipfs repo if empty") serverCmd.Flags().BoolVarP(&serverMemOnly, "mem-only", "", false, "run qri entirely in-memory, persisting nothing") serverCmd.Flags().BoolVarP(&serverOffline, "offline", "", false, "disable networking") RootCmd.AddCommand(serverCmd)