Skip to content

Commit

Permalink
fix(CreateDataset): user-specified data should override transforms
Browse files Browse the repository at this point in the history 
closes #456
  • Loading branch information
@b5
b5 committed Jun 12, 2018
1 parent d2dc519 commit dc2b0f9
Show file tree
Hide file tree
Showing 9 changed files with 217 additions and 102 deletions.
67 changes: 34 additions & 33 deletions cmd/cmd_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,10 @@ func confirmQriNotRunning() error {
return nil
}

func executeCommand(root *cobra.Command, args ...string) (output string, err error) {
func executeCommand(root *cobra.Command, cmd string) (output string, err error) {
cmd = strings.TrimPrefix(cmd, "qri ")
// WARNING - currently doesn't support quoted strings as input
args := strings.Split(cmd, " ")
_, output, err = executeCommandC(root, args...)
return output, err
}
Expand Down Expand Up @@ -142,50 +145,48 @@ func TestCommandsIntegration(t *testing.T) {
return
}

commands := [][]string{
{"help"},
{"version"},
{"setup", "--peername=" + "alan", "--registry=" + registryServer.URL},
{"config", "get", "-c"},
{"config", "get", "profile"},
{"config", "set", "webapp.port", "3505"},
// TODO - add setting whole config via a file
// {"config", "set", "-i" + profileDataFilepath},
{"info", "me"},
{"add", "--data=" + moviesFilePath, "me/movies"},
{"add", "--data=" + movies2FilePath, "me/movies2"},
{"add", "--data=" + linksFilepath, "me/links"},
{"info", "me/movies"},
{"list"},
{"save", "--data=" + movies2FilePath, "-t" + "commit_1", "me/movies"},
{"log", "me/movies"},
{"diff", "me/movies", "me/movies2", "-d", "detail"},
{"export", "-o" + path, "me/movies"},
{"export", "-o" + path, "--format=cbor", "--body-format=json", "me/movies"},
{"registry", "unpublish", "me/movies"},
{"registry", "publish", "me/movies"},
{"rename", "me/movies", "me/movie"},
{"data", "--limit=1", "--data-format=cbor", "me/movie"},
{"validate", "me/movie"},
{"remove", "me/movie"},
{"export", "--blank", "-o" + path + "/blank_dataset.yaml"},
{"setup", "--remove"},
commands := []string{
"qri help",
"qri version",
"qri setup --peername=alan --registry=" + registryServer.URL,
"qri config get -c",
"qri config get profile",
"qri config set webapp.port 3505",
"qri info me",
"qri add --data=" + moviesFilePath + " me/movies",
"qri add --data=" + movies2FilePath + " me/movies2",
"qri add --data=" + linksFilepath + " me/links",
"qri info me/movies",
"qri list",
"qri save --data=" + movies2FilePath, "-t commit_1 me/movies",
"qri log me/movies",
"qri diff me/movies me/movies2 -d=detail",
"qri export -o=" + path + " me/movies",
"qri export -o=" + path + " --format=cbor --body-format=json me/movies",
"qri registry unpublish me/movies",
"qri registry publish me/movies",
"qri rename me/movies me/movie",
"qri data --limit=1 --data-format=cbor me/movie",
"qri validate", "me/movie",
"qri remove", "me/movie",
"qri export --blank -o=" + path + "/blank_dataset.yaml",
"qri setup --remove",
}

_, in, out, err := NewTestIOStreams()
root := NewQriCommand(NewDirPathFactory(path), in, out, err)

for i, args := range commands {
for i, command := range commands {
func() {
defer func() {
if e := recover(); e != nil {
t.Errorf("case %d unexpected panic executing command\n%s\n%s", i, strings.Join(args, " "), e)
t.Errorf("case %d unexpected panic executing command\n%s\n%s", i, command, e)
return
}
}()
_, err := executeCommand(root, args...)
_, err := executeCommand(root, command)
if err != nil {
t.Errorf("case %d unexpected error executing command\n%s\n%s", i, strings.Join(args, " "), err.Error())
t.Errorf("case %d unexpected error executing command\n%s\n%s", i, command, err.Error())
return
}
}()
Expand Down
7 changes: 3 additions & 4 deletions cmd/connect_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package cmd
import (
"os"
"path/filepath"
"strings"
"testing"

regmock "github.com/qri-io/registry/regserver/mock"
Expand Down Expand Up @@ -37,13 +36,13 @@ func TestConnect(t *testing.T) {
// }
// }()

args := []string{"connect", "--setup", "--registry=" + registryServer.URL, "--disconnect-after=3"}
cmd := "qri connect --setup --registry=" + registryServer.URL + " --disconnect-after=3"
_, in, out, errs := NewTestIOStreams()
root := NewQriCommand(NewDirPathFactory(path), in, out, errs)

_, err := executeCommand(root, args...)
_, err := executeCommand(root, cmd)
if err != nil {
t.Errorf("unexpected error executing command\n%s\n%s", strings.Join(args, " "), err.Error())
t.Errorf("unexpected error executing command\n%s\n%s", cmd, err.Error())
return
}
}
5 changes: 2 additions & 3 deletions cmd/print.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ import (
"github.com/olekukonko/tablewriter"
"github.com/qri-io/dataset"
"github.com/qri-io/qri/config"
"github.com/qri-io/qri/core"
"github.com/qri-io/qri/repo"
"github.com/spf13/cobra"
)
Expand All @@ -35,8 +34,8 @@ const (
// var printPrompt = color.New(color.FgWhite).PrintfFunc()
var spinner = sp.New(sp.CharSets[24], 100*time.Millisecond)

func setNoColor() {
color.NoColor = core.Config.CLI.ColorizeOutput
func setNoColor(noColor bool) {
color.NoColor = noColor
}

func printSuccess(w io.Writer, msg string, params ...interface{}) {
Expand Down
8 changes: 6 additions & 2 deletions cmd/qri.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,9 +41,10 @@ func NewQriCommand(pf PathFactory, in io.Reader, out, err io.Writer) *cobra.Comm

// TODO: write a test that verifies this works with our new yaml config
// RootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $QRI_PATH/config.yaml)")
// RootCmd.PersistentFlags().BoolVarP(&noColor, "no-color", "c", false, "disable colorized output")
cmd.SetUsageTemplate(rootUsageTemplate)
cmd.PersistentFlags().BoolVarP(&opt.NoPrompt, "no-prompt", "", false, "disable all interactive prompts")
cmd.Flags().BoolVarP(&opt.NoPrompt, "no-prompt", "", false, "disable all interactive prompts")
cmd.Flags().BoolVarP(&opt.NoColor, "no-color", "", false, "disable colorized output")

cmd.AddCommand(
NewAddCommand(opt, ioStreams),
NewConfigCommand(opt, ioStreams),
Expand Down Expand Up @@ -82,6 +83,8 @@ type QriOptions struct {
ipfsFsPath string
// NoPrompt Disables all promt messages
NoPrompt bool
// NoColor disables colorized output
NoColor bool
// path to configuration object
ConfigPath string

Expand Down Expand Up @@ -113,6 +116,7 @@ func (o *QriOptions) init() (err error) {
return
}
o.config = core.Config
setNoColor(o.config.CLI.ColorizeOutput || o.NoColor)

addr := fmt.Sprintf(":%d", o.config.RPC.Port)
if conn, err := net.Dial("tcp", addr); err != nil {
Expand Down
2 changes: 1 addition & 1 deletion cmd/setup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ overwrite this info.`,
cmd.Flags().StringVarP(&o.Registry, "registry", "", "", "override default registry URL")
cmd.Flags().StringVarP(&o.Peername, "peername", "", "", "choose your desired peername")
cmd.Flags().StringVarP(&o.IPFSConfigData, "ipfs-config", "", "", "json-encoded configuration data, specify a filepath with '@' prefix")
cmd.Flags().StringVarP(&o.ConfigData, "conifg-data", "", "", "json-encoded configuration data, specify a filepath with '@' prefix")
cmd.Flags().StringVarP(&o.ConfigData, "config-data", "", "", "json-encoded configuration data, specify a filepath with '@' prefix")

return cmd
}
Expand Down
125 changes: 73 additions & 52 deletions cmd/tutorials_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,69 +7,85 @@ import (
"path/filepath"
"strings"
"testing"
"time"

"github.com/qri-io/dataset/dsfs"
regmock "github.com/qri-io/registry/regserver/mock"
)

type subSection struct {
name string
files []file
commands []command
}

type file struct {
name string
contents string
}

type command struct {
input string
response string
}

func TestTutorialSkylarkTransformSecOne(t *testing.T) {
testCommandSection(t, []subSection{
subSection{
name: "1.0",
files: map[string]string{
"dataset.yaml": `name: hello_world
files: []file{
{"dataset.yaml",
`name: hello_world
meta:
title: hello world example
transform:
scriptpath: $PATH/transform.sky
`,
"transform.sky": `
def transform(qri):
return(["hello", "world"])
`,
`},
{"transform.sky",
`def transform(qri):
return(["hello", "world"])
`},
},
commands: map[string]string{
"qri add --file=$PATH/dataset.yaml": "",
"qri info me/hello_world": "",
"qri data me/hello_world": "",
"qri data -s 1 me/hello_world": "",
commands: []command{
{"qri add --file=$PATH/dataset.yaml", "added new dataset me/hello_world@QmTzKAQNUL4PePZj1AupBYHm2ZwSpjXhbU5Aa9THRS5Dth/ipfs/Qmbgcpxi8gPQegxoVryoS9YBsCq7EAawhtbapy3NnFZk2A\n"},
// {"qri info me/hello_world", ""},
// {"qri data me/hello_world", ""},
// {"qri data -s 1 me/hello_world", ""},
},
},

subSection{
name: "1.1",
files: map[string]string{
"transform.sky": `
def transform(qri):
qri.set_meta("description", "this is an example dataset to learn about transformations")
return(["hello","world"])
`,
},
commands: map[string]string{
"qri update --file=$PATH/dataset.yaml": "",
"qri info me/hello_world": "",
"qri log me/hello_world": "",
},
},
// subSection{
// name: "1.1",
// files: map[string]string{
// "transform.sky": `
// def transform(qri):
// qri.set_meta("description", "this is an example dataset to learn about transformations")
// return(["hello","world"])
// `,
// },
// commands: map[string]string{
// "qri update --file=$PATH/dataset.yaml": "",
// "qri info me/hello_world": "",
// "qri log me/hello_world": "",
// },
// },
})
}

type subSection struct {
name string
commands, files map[string]string
}
const configData = `{"Profile":{"id":"QmVkB9Q9Twaqj2pHAsw3PUcZEfMngbVHK3ktxRLs5vxLVP","privkey":"CAASqAkwggSkAgEAAoIBAQDNOh8ew7u1B5zI7FzL2K1CH9ylG73OwILdMgrdMWfUWQRyv7/B53gR8tihNH+JSopIuPOZU4Hq4XlytH2tqov0nuVaD6S9yKfmRjDtx67zF8SdbOUua8QVyxJUKtgfavwzAeQrMcY3YU1xIPtLV7BZZNcCO0SR/e+/zDrkj/rCIFE0Kqvp6KlvB/T4zsruoqlMSicOnuR1qscVjcKS5BKfihhkdu/N52d02DuMpjc1Zvn2eUG8GwujoXhxTYrrw+Ahsdnjf6jD3hhPF/Tl7qF0DmKIBNqAxQURnRlzv/QmVKB+IvETs0ZgzvICx9dkcAbJD8J58jWOtcJGaS4xDPZ/AgMBAAECggEBAMz1+xaSjOjjTWYtCRBcucyMXaRX7FyMTyXyUH9QVE3zCUpAzpYtQfFLA6cb8zN+SlUOwdqjV+Doho5UTCAhb91Cw/QKI5GW0HheKYeQId2OI0LF+q0923uYcBm3IFlgU7ftb57j+v5cIyeUtOQuE2V0HHeSEY1sewCmSzrxwJIWQYT72ZgwmwUcDwZAECIWoG1OdDGSqyN+rvBaVIKzVtSVxjBJCzMIpMk+bizGswT0U972wa7k88tQR1JDqy6fsjqVu5/moHBtyZJYYX4xqbOyT84zzYmBhrxDR+LysxdoxlHhHNErmubbRcifsndKtQl3D9beg/hFYUnWDz3mTcECgYEA9hizqdjZzbiQ49RjyJE3Go2PULSk3/ywJlF0ywMKW8+hno97fHJpDhW8c+3/+RuMRfMExrmCIjtRKJAKGYiL6sjeAvXCOlkMk7bNdKhiA0QbdGDH7aNgq790cSbpEZZPjckZg8FaEkdkZMHnoqR6DWbBmqRIRV2ybxbxWl8Y1/kCgYEA1XxhcGk5loYMGej8a7HUfqmE/y9ZKvtznggqCExPs/8PV8/CgS/0ODpYgB/7zmGK4P7sXbNURDHWXTsZ7lHrZ1/2N//1KFzGrwoV4yNs4+F1KWIopnj8VvOXnI7B7aWMRVb/Iz/CrqMpk0r2CNKO5zwti9SLN0WMtrToQ6KvEDcCgYBmcEHZIXsINxuYuoAOrOPTGbneTlOTKQvp515GYy2cqgEMj41HcBhq1IFtgFRczSrGszH5xp50wVw3Fj4eAsi7IoHHmaUPxKGHm0BKnwytb1w+RUrzkF5dH4TI1PmPaJIU4vDL92AW28DecBiLiwuDKJLOXDP5K9dqhpJQm+UrgQKBgCdnDZTx1u8xTop5th0qdU3re+CdnJeSUYSZfSiK7SsLqIlMj1nIwAqgujTrsk4lMrzUUEv7DVKI4EfU4LRb6lbqH0e2tkj81q737ZI+t5pXwogg1d/j9K8XxqPetWxnYbxqkiyvV/9jngjRjReTxUxojv9/+zH4ZEBUo4nYPPeBAoGBANbeMKeelKjSjm19s6kEitykYJKpHZxP0jOspTaUuuN1MlU868RBRPq0vDuVRvxp65kSuTAxQEuvnQ8MNdC1FlL1cG/zsh68vyK+Mm/3qQjTPPR2Tkya7xrzL2Rs6v48fp43eE4Ou0CuZ1X2LQsRgODZxFfxtyWJd0smOyFPXMj6","peername":"raspberry_maltese","created":"2018-06-11T12:00:29.75648361-04:00","updated":"2018-06-11T12:00:29.75648361-04:00","type":"peer","email":"","name":"","description":"","homeurl":"","color":"","thumb":"","photo":"","poster":"","twitter":""},"Repo":{"middleware":[],"type":"fs"},"Store":{"type":"ipfs"},"P2P":{"enabled":true,"peerid":"QmbwbXJ3v7gewnF4wipkdWAa2UQGqAK3WSdYXJw8XnG1N2","pubkey":"","privkey":"CAASpwkwggSjAgEAAoIBAQC+XPf0hf1szErY3qNIF81KKjFrdhblAN8UfG8qfVlvOj9IxlrCpMenW5WvMZcFjchRSS2HZ8L8His0s0rNITe5YVAw+2ZVydNHDVyE/m8AtqIMa4hErTqhqz68WRjR9qPgKsKV1Py/JIY5ejtCL5TV+aRMpIZfeAUuzYOOnMM5wp5O+YB34fCRKp6QEpRtGQXp7JworkZRLG+RrAOyzXemUhGWuyhQ3yqlTlfeoEc+26trPs0j1rVwE0vuGrA0NGtuEAo3TmJzLTuKNbQBTEdBCs2iJJH7L6hwjfSugtJk+HuIUbuZVA93Wbh0edptmH3n/4ggP++VgvVcfhFkW42jAgMBAAECggEBAIChWXsFPyVRNhIs/rAsRS+kKxiuBvJfEF5LZuKTKOebuD94qMN1jONtY0uqep3qHD2w8O7aEktCJ3fI3vaVR44gT4uJlLDLW9RbFvVW57LXtdAmqUGevEuZ5x5mJftaaYmD21BbRx3xlDEqgvhKJowf0AFp+tJvyzDA9fBbxR81U85nCBq28lJ1uu2dRrpJ8fryVKNV0tSUND3QSM8+829nB73HiXdYux++D4F/nGlO14jOuSL7F99gwd2GKuoTDWLZIvC9YYwdk68YFROEQqHBuSoOMQN5y6sO4lLMf/1I31MXtz2XEft+SG8Xt9fhQeYS3BUNShg7EnUpcgY/VaECgYEAz1E3sTJa1ZHNT1iCy8SzATYQi8+KtNYqjSwInpGi+jkY/7K0P3XgdT574zjsD/aCb9OzdFSdS6iIaxkdGtQjhavbKWi3p7RhpTTvxyHSlwfk4eGA1K7qHyJApRd5EVVih+TXAhpeHga3vE6Xp6I7UT4EYgwR+3Pq77Y8uufsykUCgYEA6xCQLBRB0k/zEnm3fB8iOAInZw012K/ZnyRtUTMoTQP/Ml/haF1gIvsm/9poUEwBot0KWllQ+48Ga3eYOxOhrTveIbQoI8leJR3e4tb1S0cRABo5Qt9j2y5geObj6HP1s9dT6PARQSG74jf3G7JExNfsSn+yG/bU79e3htDfKscCgYBXiVuSnkhsVRvznNPdZjiPElz6K6/Nm4BsC2O4UJYYQXOkw7gH86XK249D6g0C0HtS7HihbfjcsVSoo/n+EgTNtDrdyTGCxk9Uin2T/1wEGXkzRxw2sYR5kOTq0HZ2My9fYejHIosK8psyD51JUt7o9t6y55XQLqnMfY42/SamYQKBgBB5U7FTJh3ILp/f2eUgBXTpabuGKHzsLhNCKUski7GCEzODlreKu78X3O7XbrIZ8Y9pcF1OnydSKcAEXK3fPVh3tqSkDKAdb7SoOEOrjvtcmqtoKxUttMDaqjYu8FANOG4HLEFJ+duXsqQzCeVuA/XLsCq0clYVYW0BsEIJh2pnAoGALTwBIt9TdAk/pui/4/fa7vzdcybV2hXZM4MtZfNReo8LI/FNQvDesccquvlI7tQ1he6jFUzU6trJAFfgLH8R9jri8q6SiYdaS7vBrxTnJh2dGWR4g+8yVqZdh8Xu3693eiBXKH3nXpXQaZjAIozOWs08IrfEP7yQfEvYEZpdSxk=","port":0,"addrs":null,"qribootstrapaddrs":["/ip4/130.211.198.23/tcp/4001/ipfs/QmNX9nSos8sRFvqGTwdEme6LQ8R1eJ8EuFgW32F9jjp2Pb","/ip4/35.193.162.149/tcp/4001/ipfs/QmTZxETL4YCCzB1yFx4GT1te68henVHD1XPQMkHZ1N22mm"],"httpgatewayaddr":"https://ipfs.io","profilereplication":"full","bootstrapaddrs":null},"Registry":{"location":"https://registry.qri.io"},"CLI":{"colorizeoutput":true},"API":{"enabled":true,"port":2503,"readonly":false,"urlroot":"","tls":false,"proxyforcehttps":false,"allowedorigins":["electron://local.qri.io","http://localhost:2505","http://app.qri.io","https://app.qri.io"]},"Webapp":{"enabled":true,"port":2505,"analyticstoken":"","entrypointupdateaddress":"/ipns/webapp.qri.io","entrypointhash":"Qmd7y8YyUcaDwTAhi5zxGUX2JZho5ZQstaN7qhTiWWnGR8"},"RPC":{"enabled":true,"port":2504},"Logging":{"levels":{"qriapi":"info","qrip2p":"info"}},"Render":{"templateUpdateAddress":"/ipns/defaulttmpl.qri.io","defaultTemplateHash":"/ipfs/QmeqeRTf2Cvkqdx4xUdWi1nJB2TgCyxmemsL3H4f1eTBaw"}}`

func testCommandSection(t *testing.T, subSections []subSection) {
t.Skip("not yet finished")

if err := confirmQriNotRunning(); err != nil {
t.Skip(err.Error())
}

_, registryServer := regmock.NewMockServer()
prevTs := dsfs.Timestamp
dsfs.Timestamp = func() time.Time { return time.Time{} }
defer func() { dsfs.Timestamp = prevTs }()

_, registryServer := regmock.NewMockServer()
path := filepath.Join(os.TempDir(), "qri_test_section")
t.Logf("test filepath: %s", path)
// fmt.Println(path)

//clean up if previous cleanup failed
if _, err := os.Stat(path); os.IsNotExist(err) {
Expand All @@ -79,37 +95,42 @@ func testCommandSection(t *testing.T, subSections []subSection) {
t.Errorf("error creating test path: %s", err.Error())
return
}
defer os.RemoveAll(path)

// fmt.Printf("temp path: %s", path)
os.Setenv("IPFS_PATH", filepath.Join(path, "ipfs"))
os.Setenv("QRI_PATH", filepath.Join(path, "qri"))
// defer os.RemoveAll(path)

_, in, out, errs := NewTestIOStreams()
root := NewQriCommand(NewDirPathFactory(path), in, out, errs)

// run setup
setup := fmt.Sprintf("setup --peername=alan --registry=%s", registryServer.URL)
if _, err := executeCommand(root, strings.Split(setup, " ")...); err != nil {
if _, err := executeCommand(root, fmt.Sprintf("qri setup --registry=%s --config-data=%s", registryServer.URL, configData)); err != nil {
t.Fatal(err.Error())
}
out.Reset()

// initializeCLI()
// loadConfig()
// TODO - "qri config set" will need to update the root config pointer for setting no color via a command to work properly
// for now just call internal func
setNoColor(true)
// if _, err := executeCommand(root, "qri config set cli.colorizeoutput false"); err != nil {
// t.Fatal(err.Error())
// }
// out.Reset()

for _, ss := range subSections {
for name, data := range ss.files {
data = strings.Replace(data, "$PATH", path, -1)
ioutil.WriteFile(filepath.Join(path, name), []byte(data), os.ModePerm)
for _, file := range ss.files {
data := strings.Replace(file.contents, "$PATH", path, -1)
ioutil.WriteFile(filepath.Join(path, file.name), []byte(data), os.ModePerm)
}

for cmd := range ss.commands {
cmd = strings.Replace(cmd, "$PATH", path, -1)
cmd = strings.TrimPrefix(cmd, "qri ")
_, err := executeCommand(root, strings.Split(cmd, " ")...)
if err != nil {
for _, command := range ss.commands {
cmd := strings.Replace(command.input, "$PATH", path, -1)
if _, err := executeCommand(root, cmd); err != nil {
t.Fatal(err.Error())
}

got := out.String()
if got != command.response {
t.Errorf("%s exec: %s mismatch response. expected:\n'%s'\ngot:\n'%s'", ss.name, command.input, command.response, got)
}
out.Reset()
}

}
Expand Down
Loading

0 comments on commit dc2b0f9

Please sign in to comment.