improve loading, implement diff, make models a struct/map

Signed-off-by: Tony Worm <tony@hofstadter.io>

.hof/shadow/Cli/cmd/hof/cmd/datamodel.go

@@ -25,6 +25,10 @@ func init() {
 
 	DatamodelCmd.PersistentFlags().StringSliceVarP(&(flags.DatamodelPflags.Datamodels), "datamodel", "d", nil, "Datamodels for the datamodel commands")
 	DatamodelCmd.PersistentFlags().StringSliceVarP(&(flags.DatamodelPflags.Models), "model", "m", nil, "Models for the datamodel commands")
+	DatamodelCmd.PersistentFlags().StringVarP(&(flags.DatamodelPflags.Output), "output", "o", "table", "Output format [table,cue]")
+	DatamodelCmd.PersistentFlags().StringVarP(&(flags.DatamodelPflags.Format), "format", "f", "_", "Pick format from Cuetils")
+	DatamodelCmd.PersistentFlags().StringVarP(&(flags.DatamodelPflags.Since), "since", "s", "", "Timestamp to filter since")
+	DatamodelCmd.PersistentFlags().StringVarP(&(flags.DatamodelPflags.Until), "until", "u", "", "Timestamp to filter until")
 }
 
 var DatamodelCmd = &cobra.Command{

.hof/shadow/Cli/cmd/hof/flags/datamodel.go

@@ -3,6 +3,10 @@ package flags
 type DatamodelPflagpole struct {
 	Datamodels []string
 	Models     []string
+	Output     string
+	Format     string
+	Since      string
+	Until      string
 }
 
 var DatamodelPflags DatamodelPflagpole

cmd/hof/cmd/datamodel.go

@@ -25,6 +25,10 @@ func init() {
 
 	DatamodelCmd.PersistentFlags().StringSliceVarP(&(flags.DatamodelPflags.Datamodels), "datamodel", "d", nil, "Datamodels for the datamodel commands")
 	DatamodelCmd.PersistentFlags().StringSliceVarP(&(flags.DatamodelPflags.Models), "model", "m", nil, "Models for the datamodel commands")
+	DatamodelCmd.PersistentFlags().StringVarP(&(flags.DatamodelPflags.Output), "output", "o", "table", "Output format [table,cue]")
+	DatamodelCmd.PersistentFlags().StringVarP(&(flags.DatamodelPflags.Format), "format", "f", "_", "Pick format from Cuetils")
+	DatamodelCmd.PersistentFlags().StringVarP(&(flags.DatamodelPflags.Since), "since", "s", "", "Timestamp to filter since")
+	DatamodelCmd.PersistentFlags().StringVarP(&(flags.DatamodelPflags.Until), "until", "u", "", "Timestamp to filter until")
 }
 
 var DatamodelCmd = &cobra.Command{

cmd/hof/flags/datamodel.go

@@ -3,6 +3,10 @@ package flags
 type DatamodelPflagpole struct {
 	Datamodels []string
 	Models     []string
+	Output     string
+	Format     string
+	Since      string
+	Until      string
 }
 
 var DatamodelPflags DatamodelPflagpole

design/cli/cmds/datamodel.cue

@@ -21,12 +21,40 @@ import (
 		Default: "nil"
 		Help:    "Datamodels for the datamodel commands"
 	}, {
-		Name:    "models"
+		Name:    "Models"
 		Long:    "model"
 		Short:   "m"
 		Type:    "[]string"
 		Default: "nil"
 		Help:    "Models for the datamodel commands"
+	}, {
+		Name:    "Output"
+		Long:    "output"
+		Short:   "o"
+		Type:    "string"
+		Default: "\"table\""
+		Help:    "Output format [table,cue]"
+	}, {
+		Name:    "Format"
+		Long:    "format"
+		Short:   "f"
+		Type:    "string"
+		Default: "\"_\""
+		Help:    "Pick format from Cuetils"
+	}, {
+		Name:    "Since"
+		Long:    "since"
+		Short:   "s"
+		Type:    "string"
+		Default: ""
+		Help:    "Timestamp to filter since"
+	}, {
+		Name:    "Until"
+		Long:    "until"
+		Short:   "u"
+		Type:    "string"
+		Default: ""
+		Help:    "Timestamp to filter until"
 	}]
 
 	Commands: [{

go.mod

@@ -2,6 +2,10 @@ module github.com/hofstadter-io/hof
 
 go 1.17
 
+// replace github.com/hofstadter-io/cuetils => ../../cue/cuetils
+
+replace cuelang.org/go => github.com/hofstadter-io/cue v0.4.0-hofpatch-1
+
 require (
 	cuelang.org/go v0.4.0
 	github.com/aymerick/raymond v2.0.2+incompatible
@@ -17,6 +21,7 @@ require (
 	github.com/go-git/go-billy/v5 v5.0.0
 	github.com/go-git/go-git/v5 v5.1.0
 	github.com/google/go-github/v38 v38.1.0
+	github.com/hofstadter-io/cuetils v0.3.3
 	github.com/hofstadter-io/data-utils v0.0.0-20200128210141-0a3e569b27ed
 	github.com/hofstadter-io/dotpath v0.0.0-20191027071558-52e2819b7d2d
 	github.com/hofstadter-io/hof-lang v0.0.0-20180925221847-4d1c962b0729

go.sum

@@ -10,8 +10,6 @@ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7
 cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
-cuelang.org/go v0.4.0 h1:GLJblw6m2WGGCA3k1v6Wbk9gTOt2qto48ahO2MmSd6I=
-cuelang.org/go v0.4.0/go.mod h1:tz/edkPi+T37AZcb5GlPY+WJkL6KiDlDVupKwL3vvjs=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
@@ -185,10 +183,15 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
+github.com/hofstadter-io/cue v0.4.0-hofpatch-1 h1:rlCG1LkcyK4LnFT7CGtSRYNZony/kklZa2IfWYJplUE=
+github.com/hofstadter-io/cue v0.4.0-hofpatch-1/go.mod h1:tz/edkPi+T37AZcb5GlPY+WJkL6KiDlDVupKwL3vvjs=
+github.com/hofstadter-io/cuetils v0.3.3 h1:ES3XTaybGAEtQxEnU4wPZB1dVMtnBw8tTRzDGWyf6v8=
+github.com/hofstadter-io/cuetils v0.3.3/go.mod h1:loBxKObD56zPySAXcs2lVoavKnKXO1ioyViEu3EyumA=
 github.com/hofstadter-io/data-utils v0.0.0-20200128210141-0a3e569b27ed h1:nABRLqqwgvPElkQ97rWa6VKYaUGuimOhj+m4+QML3Zg=
 github.com/hofstadter-io/data-utils v0.0.0-20200128210141-0a3e569b27ed/go.mod h1:WEV97No6/53nUz5mPEBhMJeW4JJcw2xwZZjfPxppXLg=
 github.com/hofstadter-io/dotpath v0.0.0-20191027071558-52e2819b7d2d h1:cnUMj92riFtchfxeKB5YBm2F6/0oUsuOOffUlZQh62g=
 github.com/hofstadter-io/dotpath v0.0.0-20191027071558-52e2819b7d2d/go.mod h1:hOVbz4565GFOTrLbUiUf8IUP3Oi+vFaQJf1Zh86g7/Q=
+github.com/hofstadter-io/hof v0.5.17/go.mod h1:hhHcyZNw7jVIDC0z1sbzIhoGl1KW2gCOmmUHpeSpUjQ=
 github.com/hofstadter-io/hof-lang v0.0.0-20180925221847-4d1c962b0729 h1:rJWWoxLZNE2BZmhlrlUJYE6vXlE5s8OlLAteRhfZHQw=
 github.com/hofstadter-io/hof-lang v0.0.0-20180925221847-4d1c962b0729/go.mod h1:4dVdBwjI9Mw89zYA2aE5kKZR4VL1rcBbCR0AScQ5wkU=
 github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg=

lib/cuetils/runtime.go

@@ -36,31 +36,29 @@ func (CSO CueSyntaxOptions) MakeOpts() []cue.Option {
 
 var (
 	DefaultSyntaxOpts = CueSyntaxOptions{
-		Attributes: true,
-		Concrete: false,
+		Attributes:  true,
+		Concrete:    false,
 		Definitions: true,
-		Docs: true,
-		Hidden: true,
-		Optional: true,
+		Docs:        true,
+		Hidden:      true,
+		Optional:    true,
 	}
 )
 
 type CueRuntime struct {
-
 	Entrypoints []string
 	Workspace   string
-	FS billy.Filesystem
+	FS          billy.Filesystem
 
-	CueContext *cue.Context
-	CueConfig *load.Config
+	CueContext     *cue.Context
+	CueConfig      *load.Config
 	BuildInstances []*build.Instance
-	CueErrors []error
-	FieldOpts []cue.Option
+	CueErrors      []error
+	FieldOpts      []cue.Option
 
 	CueInstance *cue.Instance
-	CueValue cue.Value
-	Value    interface{}
-
+	CueValue    cue.Value
+	Value       interface{}
 }
 
 func (CRT *CueRuntime) ConvertToValue(in interface{}) (cue.Value, error) {
@@ -109,7 +107,9 @@ func (CRT *CueRuntime) load() (err error) {
 
 	// XXX TODO XXX
 	//  add the second arg from our runtime when implemented
-	CRT.CueContext = cuecontext.New()
+	if CRT.CueContext == nil {
+		CRT.CueContext = cuecontext.New()
+	}
 	CRT.BuildInstances = load.Instances(CRT.Entrypoints, nil)
 	for _, bi := range CRT.BuildInstances {
 		// fmt.Printf("%d: start\n", i)
@@ -138,11 +138,11 @@ func (CRT *CueRuntime) load() (err error) {
 
 		// Decode? we want to be lazy
 		/*
-		err = V.Decode(&CRT.Value)
-		if err != nil {
-			errs = append(errs, err)
-			continue
-		}
+			err = V.Decode(&CRT.Value)
+			if err != nil {
+				errs = append(errs, err)
+				continue
+			}
 		*/
 
 		// fmt.Println(i, "decoded", CRT.Value)
@@ -156,4 +156,3 @@ func (CRT *CueRuntime) load() (err error) {
 
 	return nil
 }
-

lib/datamodel/checkpoint.go

@@ -12,9 +12,6 @@ import (
 	"github.com/hofstadter-io/hof/lib/yagu"
 )
 
-// YYYYMMDDHHMMSS in Golang
-const tagFmt = "20060102150405"
-
 func RunCheckpointFromArgs(args []string, flgs flags.DatamodelPflagpole) error {
 	// fmt.Println("lib/datamodel.Checkpoint", args)
 

lib/datamodel/diff.go

@@ -3,25 +3,49 @@ package datamodel
 import (
 	"fmt"
 
+	"github.com/hofstadter-io/cuetils/structural"
+
 	"github.com/hofstadter-io/hof/cmd/hof/flags"
 )
 
 func RunDiffFromArgs(args []string, flgs flags.DatamodelPflagpole) error {
-	// fmt.Println("lib/datamodel.Diff", args)
 
 	dms, err := LoadDatamodels(args, flgs)
 	if err != nil {
 		return err
 	}
 
+	dms, err = filterDatamodelsByVersion(dms, flgs)
+	if err != nil {
+		return err
+	}
+
 	for _, dm := range dms {
-		fmt.Println("---", dm.Name, "---")
 		if len(dm.History.Past) == 0 {
-			fmt.Println("no history to diff against")
+			fmt.Printf("%s: no history\n", dm.Name)
 		} else {
-			fmt.Println("compare curr to", dm.History.Past[0])
+			past := dm.History.Past[0]
+			if flgs.Since != "" {
+				past = dm.History.Past[len(dm.History.Past)-1]
+			}
+
+			fmt.Printf("// %s -> %s\n%s: ", dm.History.Past[0].version, dm.version, dm.Name)
+			diff, err := structural.DiffValue(past.value, dm.value, nil)
+			if err != nil {
+				return err
+			}
+			if !diff.Exists() {
+				fmt.Println("_|_")
+			} else {
+				fmt.Println(diff)
+			}
 		}
 	}
 
 	return nil
 }
+
+func CalcDatamodelDiff(from, to *Datamodel) error {
+
+	return nil
+}

lib/datamodel/history.go

@@ -19,7 +19,10 @@ func RunHistoryFromArgs(args []string, flgs flags.DatamodelPflagpole) error {
 		for _, ver := range dm.History.Past {
 			fmt.Println(ver.version)
 		}
-		fmt.Println()
+
+		if len(dms) > 1 {
+			fmt.Println()
+		}
 	}
 	return nil
 }

lib/datamodel/info.go

@@ -2,12 +2,91 @@ package datamodel
 
 import (
 	"fmt"
+	"regexp"
 
 	"github.com/hofstadter-io/hof/cmd/hof/flags"
+	"github.com/olekukonko/tablewriter"
 )
 
 func RunInfoFromArgs(args []string, flgs flags.DatamodelPflagpole) error {
-	fmt.Println("lib/datamodel.Status", args)
+	// fmt.Println("lib/datamodel.Info", args, flgs)
+
+	dms, err := LoadDatamodels(args, flgs)
+	if err != nil {
+		return err
+	}
+
+	dms, err = filterDatamodelsByVersion(dms, flgs)
+	if err != nil {
+		return err
+	}
+
+	return infoDatamodels(dms, flgs)
+}
+
+func infoDatamodels(dms []*Datamodel, flgs flags.DatamodelPflagpole) error {
+	switch flgs.Output {
+	case "cue":
+		return infoDatamodelsCue(dms, flgs)
+	case "table":
+		return infoDatamodelsTable(dms, flgs)
+	default:
+		return fmt.Errorf("Unknown format %q", flgs.Output)
+	}
+
+	return nil
+}
+
+func infoDatamodelsTable(dms []*Datamodel, flgs flags.DatamodelPflagpole) error {
+	return printAsTable(
+		[]string{"DM Name", "Models", "Fields", "Type", "Status"},
+		func(table *tablewriter.Table) ([][]string, error) {
+			var rows = make([][]string, 0, len(dms))
+			// fill with data
+			for _, dm := range dms {
+				dmn := dm.Name
+				for _, m := range dm.Models {
+					nf := fmt.Sprint(len(m.Fields))
+					rows = append(rows, []string{dmn, m.Name, nf, "model", m.status})
+					if len(flgs.Models) > 0 {
+						for _, f := range m.Fields {
+							rows = append(rows, []string{"", "", f.Name, f.Type, ""})
+						}
+					}
+					// only print once
+					if dmn != "" {
+						dmn = ""
+					}
+				}
+			}
+			return rows, nil
+		},
+	)
+}
+
+func infoDatamodelsCue(dms []*Datamodel, flgs flags.DatamodelPflagpole) error {
+
+	for _, dm := range dms {
+		// print whole datamodels
+		if len(flgs.Models) == 0 {
+			fmt.Printf("%s: %v\n", dm.Name, dm.value)
+			continue
+		}
+
+		// print whole models
+		if len(flgs.Models) > 0 {
+			for _, m := range dm.Models {
+				match := false
+				for _, regx := range flgs.Models {
+					if match, _ = regexp.MatchString(regx, m.Name); match {
+						fmt.Printf("%s: %s: %v\n", dm.Name, m.Name, m.value)
+					}
+				}
+			}
+			continue
+		}
+
+	}
 
 	return nil
 }