feat(qri-ref): Commands to link a working directory to a dataset

New command `qri init` links a working directory to a dataset in your repo. Then future commands will use that link instead of needing an explicit ref argument. New command `qri status` shows whether the working directory is clean. Running save will commit changes to a new version.

cmd/export.go

@@ -112,7 +112,7 @@ func (o *ExportOptions) Run() error {
 }
 
 const blankYamlDataset = `# This file defines a qri dataset. Change this file, save it, then from a terminal run:
-# $ qri add --file=dataset.yaml
+# $ qri save --file=dataset.yaml
 # For more info check out https://qri.io/docs
 
 # Name is a short name for working with this dataset without spaces for example:

cmd/init.go

@@ -1,23 +1,24 @@
 package cmd
 
 import (
+ "encoding/json"
  "fmt"
  "io/ioutil"
  "os"
  "path/filepath"
 
+ "github.com/qri-io/dataset"
  "github.com/qri-io/ioes"
  "github.com/qri-io/qri/lib"
  "github.com/spf13/cobra"
 )
 
-// NewInitCommand creates new `qri init` command that connects a directory to
-// the local filesystem
+// NewInitCommand creates new `qri init` command that connects a working directory in
+// the local filesystem to a dataset your repo.
 func NewInitCommand(f Factory, ioStreams ioes.IOStreams) *cobra.Command {
  o := &InitOptions{IOStreams: ioStreams}
  cmd := &cobra.Command{
  Use: "init",
- Aliases: []string{"ls"},
  Short: "initialize a dataset directory",
  Long: ``,
  Example: ``,
@@ -30,16 +31,18 @@ func NewInitCommand(f Factory, ioStreams ioes.IOStreams) *cobra.Command {
  },
  }
 
- cmd.PersistentFlags().StringVar(&o.Peername, "peername", "me", "ref peername")
+ cmd.PersistentFlags().StringVar(&o.Name, "name", "", "name of the dataset")
+ cmd.PersistentFlags().StringVar(&o.Format, "format", "", "format of dataset")
 
  return cmd
 }
 
-// InitOptions encapsulates state for the List command
+// InitOptions encapsulates state for the `init` command
 type InitOptions struct {
  ioes.IOStreams
 
- Peername string
+ Name string
+ Format string
  DatasetRequests *lib.DatasetRequests
 }
 
@@ -49,43 +52,136 @@ func (o *InitOptions) Complete(f Factory) (err error) {
  return err
 }
 
-// Run executes the init command
+// Run executes the `init` command
 func (o *InitOptions) Run() (err error) {
+ if _, err := os.Stat(QriRefFilename); !os.IsNotExist(err) {
+ return fmt.Errorf("working directory is already linked, .qri-ref exists")
+ }
+ if _, err := os.Stat("meta.json"); !os.IsNotExist(err) {
+ // TODO(dlong): Instead, import the meta.json file for the new dataset
+ return fmt.Errorf("cannot initialize new dataset, meta.json exists")
+ }
+ if _, err := os.Stat("schema.json"); !os.IsNotExist(err) {
+ // TODO(dlong): Instead, import the schema.json file for the new dataset
+ return fmt.Errorf("cannot initialize new dataset, schema.json exists")
+ }
+ if _, err := os.Stat("body.csv"); !os.IsNotExist(err) {
+ // TODO(dlong): Instead, import the body.csv file for the new dataset
+ return fmt.Errorf("cannot initialize new dataset, body.csv exists")
+ }
+ if _, err := os.Stat("body.json"); !os.IsNotExist(err) {
+ // TODO(dlong): Instead, import the body.json file for the new dataset
+ return fmt.Errorf("cannot initialize new dataset, body.json exists")
+ }
+
+ // Suggestion for the dataset name defaults to the name of the current directory.
  pwd, err := os.Getwd()
  if err != nil {
  return err
  }
+ suggestDataset := filepath.Base(pwd)
 
- ref := fmt.Sprintf("%s/%s", o.Peername, filepath.Base(pwd))
- path := "dataset.yaml"
+ // Process flags for inputs, prompt for any that were not provided.
+ var dsName, dsFormat string
+ if o.Name != "" {
+ dsName = o.Name
+ } else {
+ dsName = inputText(o.ErrOut, o.In, "Name of new dataset", suggestDataset)
+ }
+ if o.Format != "" {
+ dsFormat = o.Format
+ } else {
+ dsFormat = inputText(o.ErrOut, o.In, "Format of dataset, csv or json", "csv")
+ }
+
+ ref := fmt.Sprintf("me/%s", dsName)
 
+ // Validate dataset name. The `init` command must only be used for creating new datasets.
+ // Make sure a dataset with this name does not exist in your repo.
  p := lib.GetParams{
  Path: ref,
  Selector: "",
  }
-
  res := lib.GetResult{}
- if err = o.DatasetRequests.Get(&p, &res); err != nil {
- if err = ioutil.WriteFile(".qri_ref", []byte(ref), os.ModePerm); err != nil {
-  return fmt.Errorf("creating dataset reference: %s", err)
-  }
+ if err = o.DatasetRequests.Get(&p, &res); err == nil {
+ // TODO(dlong): Tell user to use `checkout` if the dataset already exists in their repo?
+ return fmt.Errorf("a dataset with the name %s already exists in your repo", ref)
+ }
 
- if _, err := os.Stat(path); os.IsNotExist(err) {
- if err := ioutil.WriteFile(path, []byte(blankYamlDataset), os.ModePerm); err != nil {
- return err
- }
- printSuccess(o.Out, "initialized qri dataset %s", path)
- return nil
+ // Validate dataset format
+ if dsFormat != "csv" && dsFormat != "json" {
+ return fmt.Errorf("invalid format \"%s\", only \"csv\" and \"json\" accepted", dsFormat)
+ }
+
+ // Create the link file, containing the dataset reference.
+ if err = ioutil.WriteFile(QriRefFilename, []byte(ref), os.ModePerm); err != nil {
+ return fmt.Errorf("creating %s file: %s", QriRefFilename, err)
+ }
+
+ // Create a skeleton meta.json file.
+ meta := dataset.Meta{
+ Qri: "md:0",
+ Citations: []*dataset.Citation{},
+ Description: "enter description here",
+ Title: "enter title here",
+ HomeURL: "enter home URL here",
+ Keywords: []string{"example"},
+ }
+ data, err := json.MarshalIndent(meta, "", " ")
+ if err := ioutil.WriteFile("meta.json", data, os.ModePerm); err != nil {
+ return err
+ }
+
+ var schema map[string]interface{}
+ if dsFormat == "csv" {
+ schema = map[string]interface{}{
+ "type": "array",
+ "items": map[string]interface{}{
+ "type": "array",
+ "items": []interface{}{
+ // First column
+ map[string]interface{}{
+ "type": "string",
+ "title": "name",
+ },
+ // Second column
+ map[string]interface{}{
+ "type": "string",
+ "title": "describe",
+ },
+ // Third column
+ map[string]interface{}{
+ "type": "integer",
+ "title": "quantity",
+ },
+ },
+ },
+ }
+ } else {
+ schema = map[string]interface{}{
+ "type": "object",
  }
- return fmt.Errorf("'%s' already exists", path)
+ }
+ data, err = json.MarshalIndent(schema, "", " ")
+ if err := ioutil.WriteFile("schema.json", data, os.ModePerm); err != nil {
+ return err
  }
 
- if _, err := os.Stat(path); os.IsNotExist(err) {
- if err := ioutil.WriteFile("dataset.yaml", res.Bytes, os.ModePerm); err != nil {
+ // Create a skeleton body file.
+ if dsFormat == "csv" {
+ bodyText := "one,two,3\nfour,five,6"
+ if err := ioutil.WriteFile("body.csv", []byte(bodyText), os.ModePerm); err != nil {
+ return err
+ }
+ } else {
+ bodyText := `{
+ "key": "value"
+}`
+ if err := ioutil.WriteFile("body.json", []byte(bodyText), os.ModePerm); err != nil {
  return err
  }
  }
 
- ref = fmt.Sprintf("%s/%s@%s%s", o.Peername, filepath.Base(pwd), res.Dataset.Commit.Author.ID, res.Dataset.Path)
- return ioutil.WriteFile(".qri_ref", []byte(ref), os.ModePerm)
+ printSuccess(o.Out, "initialized working directory for new dataset %s", ref)
+ return nil
 }

cmd/print.go

@@ -32,6 +32,10 @@ func printInfo(w io.Writer, msg string, params ...interface{}) {
  fmt.Fprintln(w, fmt.Sprintf(msg, params...))
 }
 
+func printInfoNoEndline(w io.Writer, msg string, params ...interface{}) {
+ fmt.Fprintf(w, fmt.Sprintf(msg, params...))
+}
+
 func printWarning(w io.Writer, msg string, params ...interface{}) {
  fmt.Fprintln(w, color.New(color.FgYellow).Sprintf(msg, params...))
 }
@@ -113,7 +117,7 @@ func fmtItem(i int, item string, prefix []byte) string {
 
 func prompt(w io.Writer, r io.Reader, msg string) string {
  var input string
- printInfo(w, msg)
+ printInfoNoEndline(w, msg)
  fmt.Fscanln(r, &input)
  return strings.TrimSpace(input)
 }

cmd/save.go

@@ -3,7 +3,10 @@ package cmd
 import (
  "encoding/json"
  "fmt"
+ "io/ioutil"
+ "os"
 
+ "github.com/qri-io/dataset"
  "github.com/qri-io/ioes"
  "github.com/qri-io/qfs"
  "github.com/qri-io/qri/lib"
@@ -93,15 +96,69 @@ type SaveOptions struct {
  KeepFormat bool
  Force bool
  NoRender bool
+ IsLinkedRef bool
  Secrets []string
 
  DatasetRequests *lib.DatasetRequests
 }
 
 // Complete adds any missing configuration that can only be added just before calling Run
 func (o *SaveOptions) Complete(f Factory, args []string) (err error) {
- if len(args) > 0 {
- o.Ref = args[0]
+ o.Ref, err = GetDatasetRefString(f, args, 0)
+ o.IsLinkedRef, _ = GetLinkedFilesysRef()
+
+ if o.IsLinkedRef {
+ // Determine format of the body.
+ format := ""
+ if _, err = os.Stat("body.csv"); !os.IsNotExist(err) {
+ format = "csv"
+ } else if _, err = os.Stat("body.json"); !os.IsNotExist(err) {
+ format = "json"
+ }
+ if format == "" {
+ return fmt.Errorf("could not find either body.csv or body.json")
+ }
+ o.BodyPath = fmt.Sprintf("body.%s", format)
+
+ // Read schema.
+ schema := map[string]interface{}{}
+ data, err := ioutil.ReadFile("schema.json")
+ if err != nil {
+ return err
+ }
+ if err = json.Unmarshal(data, &schema); err != nil {
+ return err
+ }
+ delete(schema, "qri")
+
+ // Create tmp structure.json
+ st := dataset.Structure{
+ Qri: "st:0",
+ Format: format,
+ Schema: schema,
+ }
+ data, err = json.Marshal(st)
+ if err != nil {
+ return err
+ }
+ tmpfile, err := ioutil.TempFile("", "example")
+ if err != nil {
+ return err
+ }
+ //defer os.Remove(tmpfile.Name())
+ if _, err := tmpfile.Write(data); err != nil {
+ return err
+ }
+ if err := tmpfile.Close(); err != nil {
+ return err
+ }
+ structureFilename := tmpfile.Name() + ".json"
+ if err = os.Rename(tmpfile.Name(), structureFilename); err != nil {
+ return err
+ }
+ // TODO: Cleanup structureFilename
+
+ o.FilePaths = []string{"meta.json", structureFilename}
  }
 
  // Make all paths absolute. Especially important if we are running
@@ -188,5 +245,11 @@ continue?`, true) {
  }
  fmt.Fprint(o.Out, string(data))
  }
+ if o.IsLinkedRef {
+ err = ioutil.WriteFile(QriRefFilename, []byte(res.String()), os.ModePerm)
+ if err != nil {
+ return err
+ }
+ }
  return nil
 }