Refine API

README.md

@@ -38,7 +38,6 @@ Each component communicates with others via GRPC and the API is defined at `api/
 - vizier: main components.
  - vizier-core : API server of vizier.
  - vizier-db
-- dlk-manager : a interface of kubernetes.
 - suggestion : implementation of each exploration algorithm.
  - vizier-suggestion-random
  - vizier-suggestion-grid

cmd/cli/get-model.go

@@ -52,68 +52,28 @@ func getModel(cmd *cobra.Command, opt *getModelOpt) {
  return
  }
  defer conn.Close()
- var soverviews []*api.StudyOverview
  c := api.NewManagerClient(conn)
  // Search study if Study ID or name is set
- if len(opt.args) > 0 {
- // Search specified study in running studies
- req := &api.GetStudiesRequest{}
- r, err := c.GetStudies(context.Background(), req)
- if err != nil {
- log.Fatalf("GetStudy failed: %v", err)
- return
- }
- if len(r.StudyInfos) > 0 {
- for _, si := range r.StudyInfos {
- if len(opt.args) > 0 {
- if utf8.RuneCountInString(opt.args[0]) >= 7 {
- if strings.HasPrefix(si.StudyId, opt.args[0]) {
- soverviews = append(soverviews, &api.StudyOverview{
- Name: si.Name,
- Owner: si.Owner,
- })
- break
- }
- }
- if si.Name == opt.args[0] {
- soverviews = append(soverviews, &api.StudyOverview{
- Name: si.Name,
- Owner: si.Owner,
- })
- break
- }
- } else {
- soverviews = append(soverviews, &api.StudyOverview{
- Name: si.Name,
- Owner: si.Owner,
- })
- }
- }
- }
+ req := &api.GetStudyListRequest{}
+ r, err := c.GetStudyList(context.Background(), req)
+ if err != nil {
+ log.Fatalf("GetModels failed: %v", err)
  }
- if len(soverviews) == 0 {
- // Search specified study from ModelDB
- sreq := &api.GetSavedStudiesRequest{}
- sr, err := c.GetSavedStudies(context.Background(), sreq)
- if err != nil {
- log.Fatalf("GetStudy failed: %v", err)
- return
- }
- if len(sr.Studies) == 0 {
- log.Fatalf("No Studies are saved.")
- return
- }
- for _, s := range sr.Studies {
- if len(opt.args) > 0 {
- if opt.args[0] == s.Name {
- soverviews = append(soverviews, s)
+ if len(r.StudyOverviews) == 0 {
+ log.Println("No Study fond")
+ return
+ }
+ for _, si := range r.StudyOverviews {
+ if len(opt.args) > 0 {
+ if utf8.RuneCountInString(opt.args[0]) >= 7 {
+ if !strings.HasPrefix(si.Id, opt.args[0]) {
+ break
  }
- } else {
- soverviews = append(soverviews, s)
+ }
+ if si.Name != opt.args[0] {
+ break
  }
  }
- }
- for _, si := range soverviews {
  // Search Models from ModelDB
  mreq := &api.GetSavedModelsRequest{StudyName: si.Name}
  mr, err := c.GetSavedModels(context.Background(), mreq)
@@ -127,11 +87,11 @@ func getModel(cmd *cobra.Command, opt *getModelOpt) {
  if opt.detail {
  for _, m := range mr.Models {
  if len(opt.args) > 1 {
- if !strings.HasPrefix(m.TrialId, opt.args[1]) {
+ if !strings.HasPrefix(m.WorkerId, opt.args[1]) {
  continue
  }
  }
- fmt.Printf("TrialID :%v\n", m.TrialId)
+ fmt.Printf("WorkerID :%v\n", m.WorkerId)
  fmt.Printf("Model Path: %s\n", m.ModelPath)
  fmt.Println("Parameters:")
  for _, p := range m.Parameters {
@@ -148,12 +108,12 @@ func getModel(cmd *cobra.Command, opt *getModelOpt) {
  fmt.Fprintln(w, "TrialID\tParamNum\tMetricsNum")
  for _, m := range mr.Models {
  if len(opt.args) > 1 {
- if !strings.HasPrefix(m.TrialId, opt.args[1]) {
+ if !strings.HasPrefix(m.WorkerId, opt.args[1]) {
  continue
  }
  }
  fmt.Fprintf(w, "%s\t%d\t%d\n",
- string([]rune(m.TrialId)[:7]),
+ string([]rune(m.WorkerId)[:7]),
  len(m.Parameters),
  len(m.Metrics),
  )

cmd/cli/get-study.go

@@ -44,40 +44,39 @@ func getStudy(cmd *cobra.Command, args []string) {
  defer conn.Close()
 
  c := api.NewManagerClient(conn)
- req := &api.GetStudiesRequest{}
- r, err := c.GetStudies(context.Background(), req)
+ req := &api.GetStudyListRequest{}
+ r, err := c.GetStudyList(context.Background(), req)
  if err != nil {
  log.Fatalf("GetStudy failed: %v", err)
  return
  }
- var sis []*api.StudyInfo
+ result := []*api.StudyOverview{}
  // Search study if Study ID or name is set
  if len(args) > 0 {
- for _, si := range r.StudyInfos {
+ for _, si := range r.StudyOverviews {
  if utf8.RuneCountInString(args[0]) >= 7 {
- if strings.HasPrefix(si.StudyId, args[0]) {
- sis = append(sis, si)
+ if strings.HasPrefix(si.Id, args[0]) {
+ result = append(result, si)
  break
  }
  }
  if si.Name == args[0] {
- sis = append(sis, si)
+ result = append(result, si)
  break
  }
  }
  } else {
- sis = r.StudyInfos
+ result = r.StudyOverviews
  }
  w := new(tabwriter.Writer)
  w.Init(os.Stdout, 0, 8, 0, '\t', tabwriter.TabIndent)
- fmt.Fprintln(w, "StudyID\tName\tOwner\tRunning\tCompleted")
- for _, si := range sis {
- fmt.Fprintf(w, "%s\t%s\t%s\t%d\t%d\n",
- string([]rune(si.StudyId)[:7]),
+ fmt.Fprintln(w, "StudyID\tName\tOwner")
+ for _, si := range result {
+ fmt.Fprintf(w, "%s\t%s\t%s\n",
+ string([]rune(si.Id)[:7]),
  si.Name,
  si.Owner,
- si.RunningTrialNum,
- si.CompletedTrialNum)
+ )
  }
  w.Flush()
 }

cmd/cli/pull-study.go

@@ -0,0 +1,105 @@
+package main
+
+import (
+ "context"
+ "fmt"
+ "io/ioutil"
+ "log"
+ "os"
+ "strings"
+ "unicode/utf8"
+
+ "github.com/kubeflow/katib/pkg/api"
+ "github.com/spf13/cobra"
+ "google.golang.org/grpc"
+ yaml "gopkg.in/yaml.v2"
+)
+
+type pullStudyOpt struct {
+ outfile string
+ args []string
+}
+
+//NewCommandPullStudy generate pull studies cmd
+func NewCommandPullStudy() *cobra.Command {
+ var opt pullStudyOpt
+ cmd := &cobra.Command{
+ Use: "studies",
+ Args: cobra.ExactArgs(1),
+ Short: "Export a Study and its Models lnfo",
+ Long: `Export Information of a Study and its Models to yaml format`,
+ Run: func(cmd *cobra.Command, args []string) {
+ opt.args = args
+ pullStudy(cmd, &opt)
+ },
+ Aliases: []string{"st"},
+ }
+ cmd.Flags().StringVarP(&opt.outfile, "output", "o", "", "File path to export")
+ return cmd
+}
+
+func pullStudy(cmd *cobra.Command, opt *pullStudyOpt) {
+ //check and get persistent flag volume
+ var pf *PersistentFlags
+ pf, err := CheckPersistentFlags()
+ if err != nil {
+ log.Fatalf("Fail to Check Flags: %v", err)
+ }
+ conn, err := grpc.Dial(pf.server, grpc.WithInsecure())
+ if err != nil {
+ log.Fatalf("could not connect: %v", err)
+ }
+ defer conn.Close()
+
+ c := api.NewManagerClient(conn)
+ listreq := &api.GetStudyListRequest{}
+ listr, err := c.GetStudyList(context.Background(), listreq)
+ if err != nil {
+ log.Fatalf("GetStudy failed: %v", err)
+ return
+ }
+ studyId := ""
+ // Search study by Study ID or name
+ for _, si := range listr.StudyOverviews {
+ if utf8.RuneCountInString(opt.args[0]) >= 7 {
+ if strings.HasPrefix(si.Id, opt.args[0]) {
+ studyId = si.Id
+ break
+ }
+ }
+ if si.Name == opt.args[0] {
+ studyId = si.Id
+ break
+ }
+ }
+ if studyId == "" {
+ log.Fatalf("Study %s is not found", opt.args[0])
+ }
+ req := &api.GetStudyRequest{
+ StudyId: studyId,
+ }
+ r, err := c.GetStudy(context.Background(), req)
+ if err != nil {
+ log.Fatalf("GetStudy failed: %v", err)
+ }
+ mreq := &api.GetSavedModelsRequest{
+ StudyName: r.StudyConfig.Name,
+ }
+ mr, err := c.GetSavedModels(context.Background(), mreq)
+ if err != nil {
+ log.Fatalf("GetModel failed: %v", err)
+ }
+ sd := StudyData{
+ StudyConf: r.StudyConfig,
+ Models: mr.Models,
+ }
+ yst, err := yaml.Marshal(sd)
+ if err != nil {
+ log.Fatalf("Failed to Marshal: %v", err)
+ }
+ if opt.outfile != "" {
+ ioutil.WriteFile(opt.outfile, yst, os.ModePerm)
+ } else {
+ fmt.Println(string(yst))
+ }
+}

cmd/cli/pull.go

@@ -0,0 +1,19 @@
+package main
+
+import (
+ "github.com/spf13/cobra"
+)
+
+//NewCommandPull generate run cmd
+func NewCommandPull() *cobra.Command {
+ cmd := &cobra.Command{
+ Use: "pull",
+ Short: "Pull a resource from a file or from stdin.",
+ Long: `YAML or JSON formats are accepted.`,
+ }
+
+ cmd.AddCommand(NewCommandPullStudy())
+ // cmd.AddCommand(NewCommandPullModel())
+
+ return cmd
+}

cmd/cli/push-model.go

@@ -14,7 +14,7 @@ import (
 )
 
 type pushModelOpt struct {
- conf string
+ file string
  args []string
 }
 
@@ -26,13 +26,13 @@ func NewCommandPushModel() *cobra.Command {
  Args: cobra.MaximumNArgs(1),
  Short: "Push a model Info from a file or from stdin",
  Long: "YAML or JSON formats are accepted.",
- Aliases: []string{"st"},
+ Aliases: []string{"md"},
  Run: func(cmd *cobra.Command, args []string) {
  opt.args = args
  pushModel(cmd, &opt)
  },
  }
- cmd.Flags().StringVarP(&opt.conf, "config", "f", "", "File path of study config")
+ cmd.Flags().StringVarP(&opt.file, "file", "f", "", "File path of model config file")
  return cmd
 }
 
@@ -44,9 +44,9 @@ func pushModel(cmd *cobra.Command, opt *pushModelOpt) {
  log.Fatalf("Fail to Check Flags: %v", err)
  return
  }
- var req api.SaveModelRequest
- if opt.conf != "" {
- buf, _ := ioutil.ReadFile(opt.conf)
+ var req []*api.SaveModelRequest
+ if opt.file != "" {
+ buf, _ := ioutil.ReadFile(opt.file)
  err = yaml.Unmarshal(buf, &req)
  if err != nil {
  log.Fatalf("Fail to Purse config: %v", err)
@@ -70,9 +70,11 @@ func pushModel(cmd *cobra.Command, opt *pushModelOpt) {
  }
  defer conn.Close()
  c := api.NewManagerClient(conn)
- _, err = c.SaveModel(context.Background(), &req)
- if err != nil {
- log.Fatalf("PushModel failed: %v", err)
+ for _, m := range req {
+ _, err = c.SaveModel(context.Background(), m)
+ if err != nil {
+ log.Fatalf("PushModel failed: %v", err)
+ }
+ fmt.Printf("Model %v is Pushed.\n", m.Model.WorkerId)
  }
- fmt.Printf("Model %v is Pushed.\n", req.Model.TrialId)
 }