From bfa9922b1b7b53b8a8411061c9d0e7df8ec0e250 Mon Sep 17 00:00:00 2001 From: Jingfu Wang Date: Sat, 9 May 2020 07:07:54 -0400 Subject: [PATCH] Dynamic registry support (#2940) * Draft PR for dynamic registry support Signed-off-by: jingfu wang * Handle migration Signed-off-by: jingfu wang * Handle migration with experimental Signed-off-by: jingfu wang * Improve error handling Signed-off-by: jingfu wang * Add unit tests Signed-off-by: jingfu wang * Fix unit tests Signed-off-by: jingfu wang * Fix unit tests Signed-off-by: jingfu wang * Add integration tests Signed-off-by: jingfu wang * Update "odo delete" and display registry name Signed-off-by: jingfu wang * Add confirmation dialog for update and delete Signed-off-by: jingfu wang * Fix catalog test Signed-off-by: jingfu wang * Update help page and delete functions Signed-off-by: jingfu wang * Help page cleanup Signed-off-by: jingfu wang * update confirmation page Signed-off-by: jingfu wang * Add URL validation Signed-off-by: jingfu wang * Fix unit test Signed-off-by: jingfu wang * Use built-in library for URL parsing Signed-off-by: jingfu wang * Update message Signed-off-by: jingfu wang * Update help page and use const default registry Signed-off-by: jingfu wang * Update registry URL Signed-off-by: jingfu wang * Update help page Signed-off-by: jingfu wang * Update registry tests Signed-off-by: jingfu wang * Add github registry example Signed-off-by: jingfu wang * Update template Signed-off-by: jingfu wang * Fix typo Signed-off-by: jingfu wang * Update k8s packages Signed-off-by: jingfu wang * Fix registry test Signed-off-by: jingfu wang --- .travis.yml | 3 +- Makefile | 5 + pkg/catalog/catalog.go | 64 ++++++-- pkg/catalog/catalog_test.go | 64 ++++++++ pkg/catalog/types.go | 10 +- pkg/odo/cli/catalog/list/components.go | 10 +- pkg/odo/cli/cli.go | 8 + pkg/odo/cli/component/create.go | 16 +- pkg/odo/cli/registry/add.go | 95 ++++++++++++ pkg/odo/cli/registry/delete.go | 86 +++++++++++ pkg/odo/cli/registry/list.go | 88 +++++++++++ pkg/odo/cli/registry/registry.go | 44 ++++++ pkg/odo/cli/registry/update.go | 92 ++++++++++++ pkg/preference/preference.go | 141 +++++++++++++++++- pkg/preference/preference_test.go | 129 ++++++++++++++++ pkg/util/util.go | 14 ++ pkg/util/util_test.go | 38 +++++ .../devfile/cmd_devfile_catalog_test.go | 23 ++- .../devfile/cmd_devfile_create_test.go | 7 + .../devfile/cmd_devfile_registry_test.go | 93 ++++++++++++ 20 files changed, 1004 insertions(+), 26 deletions(-) create mode 100644 pkg/odo/cli/registry/add.go create mode 100644 pkg/odo/cli/registry/delete.go create mode 100644 pkg/odo/cli/registry/list.go create mode 100644 pkg/odo/cli/registry/registry.go create mode 100644 pkg/odo/cli/registry/update.go create mode 100644 tests/integration/devfile/cmd_devfile_registry_test.go diff --git a/.travis.yml b/.travis.yml index 6e055582c72..68081c9cf09 100644 --- a/.travis.yml +++ b/.travis.yml @@ -130,7 +130,7 @@ jobs: # scenario of docker devfile url testing needs only Kube config file. So the test has been # added here just to make sure docker devfile url command test gets a proper kube config file. # without creating a separate OpenShift cluster. - name: "devfile catalog, create, push, delete and docker devfile url command integration tests" + name: "devfile catalog, create, push, delete, registry and docker devfile url command integration tests" script: - ./scripts/oc-cluster.sh - make bin @@ -143,6 +143,7 @@ jobs: - travis_wait make test-cmd-devfile-push - travis_wait make test-cmd-devfile-watch - travis_wait make test-cmd-devfile-delete + - travis_wait make test-cmd-devfile-registry - odo logout - <<: *base-test diff --git a/Makefile b/Makefile index 3d6e8636e7a..3914e3da1cf 100644 --- a/Makefile +++ b/Makefile @@ -217,6 +217,11 @@ test-cmd-devfile-watch: .PHONY: test-cmd-devfile-delete test-cmd-devfile-delete: ginkgo $(GINKGO_FLAGS) -focus="odo devfile delete command tests" tests/integration/devfile/ + +# Run odo devfile registry command tests +.PHONY: test-cmd-devfile-registry +test-cmd-devfile-registry: + ginkgo $(GINKGO_FLAGS) -focus="odo devfile registry command tests" tests/integration/devfile/ # Run odo storage command tests .PHONY: test-cmd-storage diff --git a/pkg/catalog/catalog.go b/pkg/catalog/catalog.go index 8a4dc6c8ab0..3f22ae744a5 100644 --- a/pkg/catalog/catalog.go +++ b/pkg/catalog/catalog.go @@ -7,7 +7,9 @@ import ( "strings" imagev1 "github.com/openshift/api/image/v1" + "github.com/openshift/odo/pkg/log" "github.com/openshift/odo/pkg/occlient" + "github.com/openshift/odo/pkg/preference" "github.com/openshift/odo/pkg/util" "github.com/pkg/errors" "gopkg.in/yaml.v2" @@ -15,10 +17,32 @@ import ( "k8s.io/klog" ) -// DevfileRegistries contains the links of all devfile registries -var DevfileRegistries = []string{ - "https://raw.githubusercontent.com/elsony/devfile-registry/master", - "https://che-devfile-registry.openshift.io/", +// GetDevfileRegistries gets devfile registries from preference file, +// if registry name is specified return the specific registry, otherwise return all registries +func GetDevfileRegistries(registryName string) (map[string]string, error) { + devfileRegistries := make(map[string]string) + + cfg, err := preference.New() + if err != nil { + return nil, err + } + + if cfg.OdoSettings.RegistryList != nil { + for _, registry := range *cfg.OdoSettings.RegistryList { + if len(registryName) != 0 { + if registryName == registry.Name { + devfileRegistries[registry.Name] = registry.URL + return devfileRegistries, nil + } + } else { + devfileRegistries[registry.Name] = registry.URL + } + } + } else { + return nil, nil + } + + return devfileRegistries, nil } // GetDevfileIndex loads the devfile registry index.json @@ -103,16 +127,26 @@ func IsDevfileComponentSupported(devfile Devfile) bool { } // ListDevfileComponents lists all the available devfile components -func ListDevfileComponents() (DevfileComponentTypeList, error) { +func ListDevfileComponents(registryName string) (DevfileComponentTypeList, error) { var catalogDevfileList DevfileComponentTypeList - catalogDevfileList.DevfileRegistries = DevfileRegistries + var err error + + // Get devfile registries + catalogDevfileList.DevfileRegistries, err = GetDevfileRegistries(registryName) + if err != nil { + return catalogDevfileList, err + } + if catalogDevfileList.DevfileRegistries == nil { + return catalogDevfileList, nil + } - for _, devfileRegistry := range DevfileRegistries { + for registryName, registryURL := range catalogDevfileList.DevfileRegistries { // Load the devfile registry index.json - devfileIndexLink := devfileRegistry + "/devfiles/index.json" + devfileIndexLink := registryURL + "/devfiles/index.json" devfileIndex, err := GetDevfileIndex(devfileIndexLink) if err != nil { - return DevfileComponentTypeList{}, err + log.Warningf("Registry %s is not set up properly with error: %v", registryName, err) + break } // 1. Load devfiles that indexed in devfile registry index.json @@ -122,14 +156,15 @@ func ListDevfileComponents() (DevfileComponentTypeList, error) { devfileIndexEntryLink := devfileIndexEntry.Links.Link // Load the devfile - devfileLink := devfileRegistry + devfileIndexEntryLink - // TODO: We send http get resquest in this function mutiple times + devfileLink := registryURL + devfileIndexEntryLink + // TODO: We send http get resquest in this function multiple times // since devfile registry uses different links to host different devfiles, // this can reduce the performance especially when we load devfiles from // big registry. We may need to rethink and optimize this in the future devfile, err := GetDevfile(devfileLink) if err != nil { - return DevfileComponentTypeList{}, err + log.Warningf("Registry %s is not set up properly with error: %v", registryName, err) + break } // Populate devfile component with devfile data and form devfile component list @@ -139,7 +174,10 @@ func ListDevfileComponents() (DevfileComponentTypeList, error) { Description: devfileIndexEntry.Description, Link: devfileIndexEntryLink, Support: IsDevfileComponentSupported(devfile), - Registry: devfileRegistry, + Registry: Registry{ + Name: registryName, + URL: registryURL, + }, } catalogDevfileList.Items = append(catalogDevfileList.Items, catalogDevfile) diff --git a/pkg/catalog/catalog_test.go b/pkg/catalog/catalog_test.go index ad0e3d44588..e201f2685ef 100644 --- a/pkg/catalog/catalog_test.go +++ b/pkg/catalog/catalog_test.go @@ -1,13 +1,16 @@ package catalog import ( + "io/ioutil" "net/http" "net/http/httptest" + "os" "reflect" "testing" imagev1 "github.com/openshift/api/image/v1" "github.com/openshift/odo/pkg/occlient" + "github.com/openshift/odo/pkg/preference" "github.com/openshift/odo/pkg/testingutil" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -168,6 +171,67 @@ func TestSliceSupportedTags(t *testing.T) { } } +func TestGetDevfileRegistries(t *testing.T) { + tempConfigFile, err := ioutil.TempFile("", "odoconfig") + if err != nil { + t.Fatal("Fail to create temporary config file") + } + defer os.Remove(tempConfigFile.Name()) + defer tempConfigFile.Close() + _, err = tempConfigFile.Write([]byte( + `kind: Preference +apiversion: odo.openshift.io/v1alpha1 +OdoSettings: + Experimental: true + RegistryList: + - Name: CheDevfileRegistry + URL: https://che-devfile-registry.openshift.io/ + - Name: DefaultDevfileRegistry + URL: https://raw.githubusercontent.com/elsony/devfile-registry/master`, + )) + if err != nil { + t.Error(err) + } + + os.Setenv(preference.GlobalConfigEnvName, tempConfigFile.Name()) + defer os.Unsetenv(preference.GlobalConfigEnvName) + + tests := []struct { + name string + registryName string + want map[string]string + }{ + { + name: "Case 1: Test get all devfile registries", + registryName: "", + want: map[string]string{ + "CheDevfileRegistry": "https://che-devfile-registry.openshift.io/", + "DefaultDevfileRegistry": "https://raw.githubusercontent.com/elsony/devfile-registry/master", + }, + }, + { + name: "Case 2: Test get specific devfile registry", + registryName: "CheDevfileRegistry", + want: map[string]string{ + "CheDevfileRegistry": "https://che-devfile-registry.openshift.io/", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := GetDevfileRegistries(tt.registryName) + if err != nil { + t.Errorf("Error message is %v", err) + } + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Got: %v, want: %v", got, tt.want) + } + }) + } +} + func TestGetDevfileIndex(t *testing.T) { // Start a local HTTP server server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { diff --git a/pkg/catalog/types.go b/pkg/catalog/types.go index a3266f0d8f6..fd7eef68916 100644 --- a/pkg/catalog/types.go +++ b/pkg/catalog/types.go @@ -12,6 +12,12 @@ type ComponentType struct { Spec ComponentSpec `json:"spec,omitempty"` } +// Registry is the main struct of devfile registry +type Registry struct { + Name string + URL string +} + // DevfileComponentType is the main struct for devfile catalog components type DevfileComponentType struct { Name string @@ -19,7 +25,7 @@ type DevfileComponentType struct { Description string Link string Support bool - Registry string + Registry Registry } // DevfileIndexEntry is the main struct of index.json from devfile registry @@ -66,7 +72,7 @@ type ComponentTypeList struct { // DevfileComponentTypeList lists all the DevfileComponentType's type DevfileComponentTypeList struct { - DevfileRegistries []string + DevfileRegistries map[string]string Items []DevfileComponentType } diff --git a/pkg/odo/cli/catalog/list/components.go b/pkg/odo/cli/catalog/list/components.go index db757f52ece..78e10fc8a02 100644 --- a/pkg/odo/cli/catalog/list/components.go +++ b/pkg/odo/cli/catalog/list/components.go @@ -57,10 +57,14 @@ func (o *ListComponentsOptions) Complete(name string, cmd *cobra.Command, args [ } if experimental.IsExperimentalModeEnabled() { - o.catalogDevfileList, err = catalog.ListDevfileComponents() + o.catalogDevfileList, err = catalog.ListDevfileComponents("") if err != nil { return err } + + if o.catalogDevfileList.DevfileRegistries == nil { + log.Warning("Please run 'odo registry add ' to add registry for listing devfile components\n") + } } return @@ -130,7 +134,7 @@ func (o *ListComponentsOptions) Run() (err error) { if len(supDevfileCatalogList) != 0 || (o.listAllDevfileComponents && len(unsupDevfileCatalogList) != 0) { fmt.Fprintln(w, "Odo Devfile Components:") - fmt.Fprintln(w, "NAME", "\t", "DESCRIPTION", "\t", "SUPPORTED") + fmt.Fprintln(w, "NAME", "\t", "DESCRIPTION", "\t", "REGISTRY", "\t", "SUPPORTED") if len(supDevfileCatalogList) != 0 { supported = "YES" @@ -192,6 +196,6 @@ func (o *ListComponentsOptions) printCatalogList(w io.Writer, catalogList []cata func (o *ListComponentsOptions) printDevfileCatalogList(w io.Writer, catalogDevfileList []catalog.DevfileComponentType, supported string) { for _, devfileComponent := range catalogDevfileList { - fmt.Fprintln(w, devfileComponent.Name, "\t", devfileComponent.Description, "\t", supported) + fmt.Fprintln(w, devfileComponent.Name, "\t", devfileComponent.Description, "\t", devfileComponent.Registry.Name, "\t", supported) } } diff --git a/pkg/odo/cli/cli.go b/pkg/odo/cli/cli.go index 6c0876f113f..d27e8c05f14 100644 --- a/pkg/odo/cli/cli.go +++ b/pkg/odo/cli/cli.go @@ -16,6 +16,7 @@ import ( "github.com/openshift/odo/pkg/odo/cli/plugins" "github.com/openshift/odo/pkg/odo/cli/preference" "github.com/openshift/odo/pkg/odo/cli/project" + "github.com/openshift/odo/pkg/odo/cli/registry" "github.com/openshift/odo/pkg/odo/cli/service" "github.com/openshift/odo/pkg/odo/cli/storage" "github.com/openshift/odo/pkg/odo/cli/url" @@ -23,6 +24,7 @@ import ( "github.com/openshift/odo/pkg/odo/cli/version" "github.com/openshift/odo/pkg/odo/util" odoutil "github.com/openshift/odo/pkg/odo/util" + "github.com/openshift/odo/pkg/odo/util/experimental" "github.com/spf13/cobra" "github.com/spf13/pflag" @@ -197,6 +199,12 @@ func odoRootCmd(name, fullName string) *cobra.Command { debug.NewCmdDebug(debug.RecommendedCommandName, util.GetFullName(fullName, debug.RecommendedCommandName)), ) + if experimental.IsExperimentalModeEnabled() { + rootCmd.AddCommand( + registry.NewCmdRegistry(registry.RecommendedCommandName, util.GetFullName(fullName, registry.RecommendedCommandName)), + ) + } + odoutil.VisitCommands(rootCmd, reconfigureCmdWithSubcmd) return rootCmd diff --git a/pkg/odo/cli/component/create.go b/pkg/odo/cli/component/create.go index 2f8672a2583..02c2863041c 100644 --- a/pkg/odo/cli/component/create.go +++ b/pkg/odo/cli/component/create.go @@ -63,7 +63,7 @@ type DevfileMetadata struct { componentNamespace string devfileSupport bool devfileLink string - devfileRegistry string + devfileRegistry catalog.Registry downloadSource bool } @@ -329,10 +329,13 @@ func (co *CreateOptions) Complete(name string, cmd *cobra.Command, args []string co.CommonPushOptions.componentContext = co.componentContext } - catalogDevfileList, err := catalog.ListDevfileComponents() + catalogDevfileList, err := catalog.ListDevfileComponents(co.devfileMetadata.devfileRegistry.Name) if err != nil { return err } + if catalogDevfileList.DevfileRegistries == nil { + log.Warning("Please run `odo registry add ` to add a registry then create a devfile components\n") + } var componentType string var componentName string @@ -437,6 +440,8 @@ func (co *CreateOptions) Complete(name string, cmd *cobra.Command, args []string } } + registrySpinner := log.Spinnerf("Creating a devfile component from registry: %s", co.devfileMetadata.devfileRegistry.Name) + if co.devfileMetadata.devfileSupport { err = co.InitEnvInfoFromContext() if err != nil { @@ -444,11 +449,13 @@ func (co *CreateOptions) Complete(name string, cmd *cobra.Command, args []string } spinner.End(true) + registrySpinner.End(true) return nil } spinner.End(false) - log.Italic("\nPlease run 'odo catalog list components' for a list of supported devfile component types") + registrySpinner.End(false) + log.Italic("\nPlease run `odo catalog list components` for a list of supported devfile component types") } if len(args) == 0 || !cmd.HasFlags() { @@ -773,7 +780,7 @@ func (co *CreateOptions) Run() (err error) { // Download devfile.yaml file and create env.yaml file if co.devfileMetadata.devfileSupport { if !util.CheckPathExists(DevfilePath) { - err := util.DownloadFile(co.devfileMetadata.devfileRegistry+co.devfileMetadata.devfileLink, DevfilePath) + err := util.DownloadFile(co.devfileMetadata.devfileRegistry.URL+co.devfileMetadata.devfileLink, DevfilePath) if err != nil { return errors.Wrap(err, "Faile to download devfile.yaml for devfile component") } @@ -899,6 +906,7 @@ func NewCmdCreate(name, fullName string) *cobra.Command { componentCreateCmd.Flags().StringSliceVar(&co.componentEnvVars, "env", []string{}, "Environmental variables for the component. For example --env VariableName=Value") if experimental.IsExperimentalModeEnabled() { + componentCreateCmd.Flags().StringVar(&co.devfileMetadata.devfileRegistry.Name, "registry", "", "Create devfile component from specific registry") componentCreateCmd.Flags().BoolVar(&co.devfileMetadata.downloadSource, "downloadSource", false, "Download sample project from devfile. (ex. odo component create [component_name] --downloadSource") } diff --git a/pkg/odo/cli/registry/add.go b/pkg/odo/cli/registry/add.go new file mode 100644 index 00000000000..ca891b3c391 --- /dev/null +++ b/pkg/odo/cli/registry/add.go @@ -0,0 +1,95 @@ +package registry + +import ( + // Built-in packages + "fmt" + + // Third-party packages + "github.com/pkg/errors" + "github.com/spf13/cobra" + ktemplates "k8s.io/kubectl/pkg/util/templates" + + // odo packages + "github.com/openshift/odo/pkg/log" + "github.com/openshift/odo/pkg/odo/genericclioptions" + "github.com/openshift/odo/pkg/preference" + "github.com/openshift/odo/pkg/util" +) + +const addCommandName = "add" + +// "odo registry add" command description and examples +var ( + addLongDesc = ktemplates.LongDesc(`Add devfile registry`) + + addExample = ktemplates.Examples(`# Add devfile registry + %[1]s CheRegistry https://che-devfile-registry.openshift.io + + %[1]s CheRegistryFromGitHub https://raw.githubusercontent.com/eclipse/che-devfile-registry/master + `) +) + +// AddOptions encapsulates the options for the "odo registry add" command +type AddOptions struct { + operation string + registryName string + registryURL string + forceFlag bool +} + +// NewAddOptions creates a new AddOptions instance +func NewAddOptions() *AddOptions { + return &AddOptions{} +} + +// Complete completes AddOptions after they've been created +func (o *AddOptions) Complete(name string, cmd *cobra.Command, args []string) (err error) { + o.operation = "add" + o.registryName = args[0] + o.registryURL = args[1] + return +} + +// Validate validates the AddOptions based on completed values +func (o *AddOptions) Validate() (err error) { + err = util.ValidateURL(o.registryURL) + if err != nil { + return err + } + + return +} + +// Run contains the logic for "odo registry add" command +func (o *AddOptions) Run() (err error) { + + cfg, err := preference.New() + if err != nil { + return errors.Wrap(err, "unable to add registry") + } + + err = cfg.RegistryHandler(o.operation, o.registryName, o.registryURL, o.forceFlag) + if err != nil { + return err + } + + log.Info("New registry successfully added") + return nil +} + +// NewCmdAdd implements the "odo registry add" command +func NewCmdAdd(name, fullName string) *cobra.Command { + o := NewAddOptions() + registryAddCmd := &cobra.Command{ + Use: fmt.Sprintf("%s ", name), + Short: addLongDesc, + Long: addLongDesc, + Example: fmt.Sprintf(fmt.Sprint(addExample), fullName), + Args: cobra.ExactArgs(2), + Run: func(cmd *cobra.Command, args []string) { + genericclioptions.GenericRun(o, cmd, args) + }, + } + + return registryAddCmd +} diff --git a/pkg/odo/cli/registry/delete.go b/pkg/odo/cli/registry/delete.go new file mode 100644 index 00000000000..4d3ea5c00f6 --- /dev/null +++ b/pkg/odo/cli/registry/delete.go @@ -0,0 +1,86 @@ +package registry + +import ( + // Built-in packages + "fmt" + + // Third-party packages + "github.com/pkg/errors" + "github.com/spf13/cobra" + ktemplates "k8s.io/kubectl/pkg/util/templates" + + // odo packages + "github.com/openshift/odo/pkg/odo/genericclioptions" + "github.com/openshift/odo/pkg/preference" +) + +const deleteCommandName = "delete" + +// "odo registry delete" command description and examples +var ( + deleteLongDesc = ktemplates.LongDesc(`Delete devfile registry`) + + deleteExample = ktemplates.Examples(`# Delete devfile registry + %[1]s CheRegistry + `) +) + +// DeleteOptions encapsulates the options for the "odo registry delete" command +type DeleteOptions struct { + operation string + registryName string + registryURL string + forceFlag bool +} + +// NewDeleteOptions creates a new DeleteOptions instance +func NewDeleteOptions() *DeleteOptions { + return &DeleteOptions{} +} + +// Complete completes DeleteOptions after they've been created +func (o *DeleteOptions) Complete(name string, cmd *cobra.Command, args []string) (err error) { + o.operation = "delete" + o.registryName = args[0] + o.registryURL = "" + return +} + +// Validate validates the DeleteOptions based on completed values +func (o *DeleteOptions) Validate() (err error) { + return +} + +// Run contains the logic for "odo registry delete" command +func (o *DeleteOptions) Run() (err error) { + cfg, err := preference.New() + if err != nil { + return errors.Wrap(err, "unable to delete registry") + } + + err = cfg.RegistryHandler(o.operation, o.registryName, o.registryURL, o.forceFlag) + if err != nil { + return err + } + + return nil +} + +// NewCmdDelete implements the "odo registry delete" command +func NewCmdDelete(name, fullName string) *cobra.Command { + o := NewDeleteOptions() + registryDeleteCmd := &cobra.Command{ + Use: fmt.Sprintf("%s ", name), + Short: deleteLongDesc, + Long: deleteLongDesc, + Example: fmt.Sprintf(fmt.Sprint(deleteExample), fullName), + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + genericclioptions.GenericRun(o, cmd, args) + }, + } + + registryDeleteCmd.Flags().BoolVarP(&o.forceFlag, "force", "f", false, "Don't ask for confirmation, delete the registry directly") + + return registryDeleteCmd +} diff --git a/pkg/odo/cli/registry/list.go b/pkg/odo/cli/registry/list.go new file mode 100644 index 00000000000..dac98ba7414 --- /dev/null +++ b/pkg/odo/cli/registry/list.go @@ -0,0 +1,88 @@ +package registry + +import ( + // Built-in packages + "fmt" + "io" + "os" + "text/tabwriter" + + // Third-party packages + "github.com/spf13/cobra" + ktemplates "k8s.io/kubectl/pkg/util/templates" + + // odo packages + "github.com/openshift/odo/pkg/odo/genericclioptions" + "github.com/openshift/odo/pkg/odo/util" + "github.com/openshift/odo/pkg/preference" +) + +const listCommandName = "list" + +// "odo registry list" command description and examples +var ( + listDesc = ktemplates.LongDesc(`List devfile registry`) + + listExample = ktemplates.Examples(`# List devfile registry + %[1]s + `) +) + +// ListOptions encapsulates the options for "odo registry list" command +type ListOptions struct { +} + +// NewListOptions creates a new ListOptions instance +func NewListOptions() *ListOptions { + return &ListOptions{} +} + +// Complete completes ListOptions after they've been created +func (o *ListOptions) Complete(name string, cmd *cobra.Command, args []string) (err error) { + return +} + +// Validate validates the ListOptions based on completed values +func (o *ListOptions) Validate() (err error) { + return +} + +// Run contains the logic for "odo registry list" command +func (o *ListOptions) Run() (err error) { + cfg, err := preference.New() + if err != nil { + util.LogErrorAndExit(err, "") + } + + w := tabwriter.NewWriter(os.Stdout, 5, 2, 3, ' ', tabwriter.TabIndent) + fmt.Fprintln(w, "NAME", "\t", "URL") + o.printRegistryList(w, cfg.OdoSettings.RegistryList) + w.Flush() + return +} + +func (o *ListOptions) printRegistryList(w io.Writer, registryList *[]preference.Registry) { + if registryList == nil { + return + } + + for _, registry := range *registryList { + fmt.Fprintln(w, registry.Name, "\t", registry.URL) + } +} + +// NewCmdList implements the "odo registry list" command +func NewCmdList(name, fullName string) *cobra.Command { + o := NewListOptions() + registryListCmd := &cobra.Command{ + Use: name, + Short: listDesc, + Long: listDesc, + Example: fmt.Sprintf(fmt.Sprint(listExample), fullName), + Args: cobra.ExactArgs(0), + Run: func(cmd *cobra.Command, args []string) { + genericclioptions.GenericRun(o, cmd, args) + }, + } + return registryListCmd +} diff --git a/pkg/odo/cli/registry/registry.go b/pkg/odo/cli/registry/registry.go new file mode 100644 index 00000000000..a4c85bac9ff --- /dev/null +++ b/pkg/odo/cli/registry/registry.go @@ -0,0 +1,44 @@ +package registry + +import ( + // Built-in packages + "fmt" + + // Third-party packages + "github.com/spf13/cobra" + ktemplates "k8s.io/kubectl/pkg/util/templates" + + // odo packages + "github.com/openshift/odo/pkg/odo/util" +) + +// RecommendedCommandName is the recommended registry command name +const RecommendedCommandName = "registry" + +var registryDesc = ktemplates.LongDesc(`Configure devfile registry`) + +// NewCmdRegistry implements the registry configuration command +func NewCmdRegistry(name, fullName string) *cobra.Command { + registryAddCmd := NewCmdAdd(addCommandName, util.GetFullName(fullName, addCommandName)) + registryListCmd := NewCmdList(listCommandName, util.GetFullName(fullName, listCommandName)) + registryUpdateCmd := NewCmdUpdate(updateCommandName, util.GetFullName(fullName, updateCommandName)) + registryDeleteCmd := NewCmdDelete(deleteCommandName, util.GetFullName(fullName, deleteCommandName)) + + registryCmd := &cobra.Command{ + Use: name, + Short: registryDesc, + Long: registryDesc, + Example: fmt.Sprintf("%s\n\n%s\n\n%s\n\n%s", + registryAddCmd.Example, + registryListCmd.Example, + registryUpdateCmd.Example, + registryDeleteCmd.Example, + ), + } + + registryCmd.AddCommand(registryAddCmd, registryListCmd, registryUpdateCmd, registryDeleteCmd) + registryCmd.SetUsageTemplate(util.CmdUsageTemplate) + registryCmd.Annotations = map[string]string{"command": "main"} + + return registryCmd +} diff --git a/pkg/odo/cli/registry/update.go b/pkg/odo/cli/registry/update.go new file mode 100644 index 00000000000..afdab02de5a --- /dev/null +++ b/pkg/odo/cli/registry/update.go @@ -0,0 +1,92 @@ +package registry + +import ( + // Built-in packages + "fmt" + + // Third-party packages + "github.com/pkg/errors" + "github.com/spf13/cobra" + ktemplates "k8s.io/kubectl/pkg/util/templates" + + // odo packages + "github.com/openshift/odo/pkg/odo/genericclioptions" + "github.com/openshift/odo/pkg/preference" + "github.com/openshift/odo/pkg/util" +) + +const updateCommandName = "update" + +// "odo registry update" command description and examples +var ( + updateLongDesc = ktemplates.LongDesc(`Update devfile registry URL`) + + updateExample = ktemplates.Examples(`# Update devfile registry URL + %[1]s CheRegistry https://che-devfile-registry-update.openshift.io + `) +) + +// UpdateOptions encapsulates the options for the "odo registry update" command +type UpdateOptions struct { + operation string + registryName string + registryURL string + forceFlag bool +} + +// NewUpdateOptions creates a new UpdateOptions instance +func NewUpdateOptions() *UpdateOptions { + return &UpdateOptions{} +} + +// Complete completes UpdateOptions after they've been created +func (o *UpdateOptions) Complete(name string, cmd *cobra.Command, args []string) (err error) { + o.operation = "update" + o.registryName = args[0] + o.registryURL = args[1] + return +} + +// Validate validates the UpdateOptions based on completed values +func (o *UpdateOptions) Validate() (err error) { + err = util.ValidateURL(o.registryURL) + if err != nil { + return err + } + + return +} + +// Run contains the logic for "odo registry update" command +func (o *UpdateOptions) Run() (err error) { + cfg, err := preference.New() + if err != nil { + return errors.Wrap(err, "unable to update registry") + } + + err = cfg.RegistryHandler(o.operation, o.registryName, o.registryURL, o.forceFlag) + if err != nil { + return err + } + + return nil +} + +// NewCmdUpdate implements the "odo registry update" command +func NewCmdUpdate(name, fullName string) *cobra.Command { + o := NewUpdateOptions() + registryUpdateCmd := &cobra.Command{ + Use: fmt.Sprintf("%s ", name), + Short: updateLongDesc, + Long: updateLongDesc, + Example: fmt.Sprintf(fmt.Sprint(updateExample), fullName), + Args: cobra.ExactArgs(2), + Run: func(cmd *cobra.Command, args []string) { + genericclioptions.GenericRun(o, cmd, args) + }, + } + + registryUpdateCmd.Flags().BoolVarP(&o.forceFlag, "force", "f", false, "Don't ask for confirmation, update the registry directly") + + return registryUpdateCmd +} diff --git a/pkg/preference/preference.go b/pkg/preference/preference.go index 460a7c0f447..dc5542e04d7 100644 --- a/pkg/preference/preference.go +++ b/pkg/preference/preference.go @@ -12,6 +12,8 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog" + "github.com/openshift/odo/pkg/log" + "github.com/openshift/odo/pkg/odo/cli/ui" "github.com/openshift/odo/pkg/util" ) @@ -64,6 +66,18 @@ const ( // KubePushTarget represents the value of the push target when it's set to Kube KubePushTarget = "kube" + + // CheDevfileRegistryName is the name of Che devfile registry + CheDevfileRegistryName = "CheDevfileRegistry" + + // CheDevfileRegistryURL is the URL of Che devfile registry + CheDevfileRegistryURL = "https://che-devfile-registry.openshift.io" + + // DefaultDevfileRegistryName is the name of default devfile registry + DefaultDevfileRegistryName = "DefaultDevfileRegistry" + + // DefaultDevfileRegistryURL is the URL of default devfile registry + DefaultDevfileRegistryURL = "https://raw.githubusercontent.com/elsony/devfile-registry/master" ) // TimeoutSettingDescription is human-readable description for the timeout setting @@ -117,6 +131,15 @@ type OdoSettings struct { // PushTarget for telling odo which platform to push to (either kube or docker) PushTarget *string `yaml:"PushTarget,omitempty"` + + // RegistryList for telling odo to connect to all the registries in the registry list + RegistryList *[]Registry `yaml:"RegistryList,omitempty"` +} + +// Registry includes the registry metadata +type Registry struct { + Name string `yaml:"Name,omitempty"` + URL string `yaml:"URL,omitempty"` } // Preference stores all the preferences related to odo @@ -171,7 +194,8 @@ func NewPreferenceInfo() (*PreferenceInfo, error) { Preference: NewPreference(), Filename: preferenceFile, } - // if the preference file doesn't exist then we dont worry about it and return + + // If the preference file doesn't exist then we return with default preference if _, err = os.Stat(preferenceFile); os.IsNotExist(err) { return &c, nil } @@ -180,9 +204,124 @@ func NewPreferenceInfo() (*PreferenceInfo, error) { if err != nil { return nil, err } + + if c.OdoSettings.Experimental != nil && *c.OdoSettings.Experimental { + if c.OdoSettings.RegistryList == nil { + // Handle user has preference file but doesn't use dynamic registry before + defaultRegistryList := []Registry{ + { + Name: CheDevfileRegistryName, + URL: CheDevfileRegistryURL, + }, + { + Name: DefaultDevfileRegistryName, + URL: DefaultDevfileRegistryURL, + }, + } + c.OdoSettings.RegistryList = &defaultRegistryList + } + } + return &c, nil } +// RegistryHandler handles registry add, update and delete operations +func (c *PreferenceInfo) RegistryHandler(operation string, registryName string, registryURL string, forceFlag bool) error { + var registryList []Registry + var err error + registryExist := false + + // Registry list is empty + if c.OdoSettings.RegistryList == nil { + registryList, err = handleWithoutRegistryExist(registryList, operation, registryName, registryURL) + if err != nil { + return err + } + } else { + // The target registry exists in the registry list + registryList = *c.OdoSettings.RegistryList + for index, registry := range registryList { + if registry.Name == registryName { + registryExist = true + registryList, err = handleWithRegistryExist(index, registryList, operation, registryName, registryURL, forceFlag) + if err != nil { + return err + } + } + } + + // The target registry doesn't exist in the registry list + if !registryExist { + registryList, err = handleWithoutRegistryExist(registryList, operation, registryName, registryURL) + if err != nil { + return err + } + } + } + + c.OdoSettings.RegistryList = ®istryList + err = util.WriteToFile(&c.Preference, c.Filename) + if err != nil { + return errors.Wrapf(err, "unable to write the configuration of %s operation to preference file", operation) + } + + return nil +} + +func handleWithoutRegistryExist(registryList []Registry, operation string, registryName string, registryURL string) ([]Registry, error) { + switch operation { + + case "add": + registry := Registry{ + Name: registryName, + URL: registryURL, + } + registryList = append(registryList, registry) + + case "update": + return nil, errors.Errorf("failed to update registry: registry %s doesn't exist", registryName) + + case "delete": + return nil, errors.Errorf("failed to delete registry: registry %s doesn't exist", registryName) + } + + return registryList, nil +} + +func handleWithRegistryExist(index int, registryList []Registry, operation string, registryName string, registryURL string, forceFlag bool) ([]Registry, error) { + switch operation { + + case "add": + return nil, errors.Errorf("failed to add registry: registry %s already exists", registryName) + + case "update": + if !forceFlag { + if !ui.Proceed(fmt.Sprintf("Are you sure you want to update registry %s", registryName)) { + log.Info("Aborted by the user") + return registryList, nil + } + } + + registryList[index].URL = registryURL + log.Info("Successfully updated registry") + + case "delete": + if !forceFlag { + if !ui.Proceed(fmt.Sprintf("Are you sure you want to delete registry %s", registryName)) { + log.Info("Aborted by the user") + return registryList, nil + } + } + + copy(registryList[index:], registryList[index+1:]) + registryList[len(registryList)-1] = Registry{} + registryList = registryList[:len(registryList)-1] + log.Info("Successfully deleted registry") + } + + return registryList, nil +} + // SetConfiguration modifies Odo configurations in the config file // as of now being used for nameprefix, timeout, updatenotification // TODO: Use reflect to set parameters diff --git a/pkg/preference/preference_test.go b/pkg/preference/preference_test.go index 73c202c0fb4..d3b5ea22407 100644 --- a/pkg/preference/preference_test.go +++ b/pkg/preference/preference_test.go @@ -703,3 +703,132 @@ func TestMetaTypePopulatedInPreference(t *testing.T) { t.Error("the api version and kind in preference are incorrect") } } + +func TestHandleWithoutRegistryExist(t *testing.T) { + tests := []struct { + name string + registryList []Registry + operation string + registryName string + registryURL string + want []Registry + }{ + { + name: "Case 1: Add registry", + registryList: []Registry{}, + operation: "add", + registryName: "testName", + registryURL: "testURL", + want: []Registry{ + { + Name: "testName", + URL: "testURL", + }, + }, + }, + { + name: "Case 2: Update registry", + registryList: []Registry{}, + operation: "update", + registryName: "testName", + registryURL: "testURL", + want: nil, + }, + { + name: "Case 3: Delete registry", + registryList: []Registry{}, + operation: "delete", + registryName: "testName", + registryURL: "testURL", + want: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := handleWithoutRegistryExist(tt.registryList, tt.operation, tt.registryName, tt.registryURL) + if err != nil { + t.Logf("Error message is %v", err) + } + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Got: %v, want %v", got, tt.want) + } + }) + } +} + +func TestHandleWithRegistryExist(t *testing.T) { + tests := []struct { + name string + index int + registryList []Registry + operation string + registryName string + registryURL string + forceFlag bool + want []Registry + }{ + { + name: "Case 1: Add registry", + index: 0, + registryList: []Registry{ + { + Name: "testName", + URL: "testURL", + }, + }, + operation: "add", + registryName: "testName", + registryURL: "addURL", + forceFlag: false, + want: nil, + }, + { + name: "Case 2: update registry", + index: 0, + registryList: []Registry{ + { + Name: "testName", + URL: "testURL", + }, + }, + operation: "update", + registryName: "testName", + registryURL: "updateURL", + forceFlag: true, + want: []Registry{ + { + Name: "testName", + URL: "updateURL", + }, + }, + }, + { + name: "Case 3: Delete registry", + index: 0, + registryList: []Registry{ + { + Name: "testName", + URL: "testURL", + }, + }, + operation: "delete", + registryName: "testName", + registryURL: "", + forceFlag: true, + want: []Registry{}, + }, + } + + for _, tt := range tests { + got, err := handleWithRegistryExist(tt.index, tt.registryList, tt.operation, tt.registryName, tt.registryURL, tt.forceFlag) + if err != nil { + t.Logf("Error message is %v", err) + } + + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Got: %v, want: %v", got, tt.want) + } + } +} diff --git a/pkg/util/util.go b/pkg/util/util.go index 2dce6cf36c6..8d760d1aac5 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -982,3 +982,17 @@ func CheckKubeConfigExist() bool { return false } + +// ValidateURL validates the URL +func ValidateURL(sourceURL string) error { + u, err := url.Parse(sourceURL) + if err != nil { + return err + } + + if len(u.Host) == 0 || len(u.Scheme) == 0 { + return errors.New("URL is invalid") + } + + return nil +} diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index 94f5b88c941..1dae55a141f 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -1614,3 +1614,41 @@ func TestGetGitHubZipURL(t *testing.T) { } } */ + +func TestValidateURL(t *testing.T) { + tests := []struct { + name string + url string + wantErr bool + }{ + { + name: "Case 1: Valid URL", + url: "http://www.example.com/", + wantErr: false, + }, + { + name: "Case 2: Invalid URL - No host", + url: "http://", + wantErr: true, + }, + { + name: "Case 3: Invalid URL - No scheme", + url: "://www.example.com/", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotErr := false + got := ValidateURL(tt.url) + if got != nil { + gotErr = true + } + + if !reflect.DeepEqual(gotErr, tt.wantErr) { + t.Errorf("Got %v, want %v", got, tt.wantErr) + } + }) + } +} diff --git a/tests/integration/devfile/cmd_devfile_catalog_test.go b/tests/integration/devfile/cmd_devfile_catalog_test.go index 4b9c0cdebda..d2aa5f7a85a 100644 --- a/tests/integration/devfile/cmd_devfile_catalog_test.go +++ b/tests/integration/devfile/cmd_devfile_catalog_test.go @@ -48,14 +48,33 @@ var _ = Describe("odo devfile catalog command tests", func() { Context("When executing catalog list components", func() { It("should list all supported devfile components", func() { output := helper.CmdShouldPass("odo", "catalog", "list", "components") - helper.MatchAllInOutput(output, []string{"Odo Devfile Components", "java-spring-boot", "openLiberty"}) + wantOutput := []string{ + "Odo Devfile Components", + "NAME", + "java-spring-boot", + "openLiberty", + "DESCRIPTION", + "REGISTRY", + "SUPPORTED", + } + helper.MatchAllInOutput(output, wantOutput) }) }) Context("When executing catalog list components with -a flag", func() { It("should list all supported and unsupported devfile components", func() { output := helper.CmdShouldPass("odo", "catalog", "list", "components", "-a") - helper.MatchAllInOutput(output, []string{"Odo Devfile Components", "java-spring-boot", "java-maven", "php-mysql"}) + wantOutput := []string{ + "Odo Devfile Components", + "NAME", + "java-spring-boot", + "java-maven", + "php-mysql", + "DESCRIPTION", + "REGISTRY", + "SUPPORTED", + } + helper.MatchAllInOutput(output, wantOutput) }) }) }) diff --git a/tests/integration/devfile/cmd_devfile_create_test.go b/tests/integration/devfile/cmd_devfile_create_test.go index e9e58702a59..0bfdc16bfac 100644 --- a/tests/integration/devfile/cmd_devfile_create_test.go +++ b/tests/integration/devfile/cmd_devfile_create_test.go @@ -79,6 +79,13 @@ var _ = Describe("odo devfile create command tests", func() { }) }) + Context("When executing odo create with devfile component type argument and --registry flag", func() { + It("should successfully create the devfile component", func() { + componentRegistry := "DefaultDevfileRegistry" + helper.CmdShouldPass("odo", "create", "openLiberty", "--registry", componentRegistry) + }) + }) + Context("When executing odo create with devfile component type argument and --context flag", func() { It("should successfully create the devfile component in the context", func() { newContext := path.Join(context, "newContext") diff --git a/tests/integration/devfile/cmd_devfile_registry_test.go b/tests/integration/devfile/cmd_devfile_registry_test.go new file mode 100644 index 00000000000..85c27e0297c --- /dev/null +++ b/tests/integration/devfile/cmd_devfile_registry_test.go @@ -0,0 +1,93 @@ +package devfile + +import ( + "os" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/openshift/odo/tests/helper" +) + +var _ = Describe("odo devfile registry command tests", func() { + var project string + var context string + var currentWorkingDirectory string + const registryName string = "RegistryName" + const addRegistryURL string = "https://raw.githubusercontent.com/GeekArthur/registry/master" + const updateRegistryURL string = "http://www.example.com/update" + + // This is run after every Spec (It) + var _ = BeforeEach(func() { + SetDefaultEventuallyTimeout(10 * time.Minute) + context = helper.CreateNewContext() + os.Setenv("GLOBALODOCONFIG", filepath.Join(context, "config.yaml")) + helper.CmdShouldPass("odo", "preference", "set", "Experimental", "true") + if os.Getenv("KUBERNETES") == "true" { + project = helper.CreateRandNamespace(context) + } else { + project = helper.CreateRandProject() + } + currentWorkingDirectory = helper.Getwd() + helper.Chdir(context) + }) + + // This is run after every Spec (It) + var _ = AfterEach(func() { + if os.Getenv("KUBERNETES") == "true" { + helper.DeleteNamespace(project) + } else { + helper.DeleteProject(project) + } + helper.Chdir(currentWorkingDirectory) + helper.DeleteDir(context) + }) + + Context("When executing registry list", func() { + It("Should list all default registries", func() { + output := helper.CmdShouldPass("odo", "registry", "list") + helper.MatchAllInOutput(output, []string{"CheDevfileRegistry", "DefaultDevfileRegistry"}) + }) + }) + + Context("When executing registry commands with the registry is not present", func() { + It("Should successfully add the registry", func() { + helper.CmdShouldPass("odo", "registry", "add", registryName, addRegistryURL) + output := helper.CmdShouldPass("odo", "registry", "list") + helper.MatchAllInOutput(output, []string{registryName, addRegistryURL}) + helper.CmdShouldPass("odo", "create", "nodejs", "--registry", registryName) + helper.CmdShouldPass("odo", "registry", "delete", registryName, "-f") + }) + + It("Should fail to update the registry", func() { + helper.CmdShouldFail("odo", "registry", "update", registryName, updateRegistryURL, "-f") + }) + + It("Should fail to delete the registry", func() { + helper.CmdShouldFail("odo", "registry", "delete", registryName, "-f") + }) + }) + + Context("When executing registry commands with the registry is present", func() { + It("Should fail to add the registry", func() { + helper.CmdShouldPass("odo", "registry", "add", registryName, addRegistryURL) + helper.CmdShouldFail("odo", "registry", "add", registryName, addRegistryURL) + helper.CmdShouldPass("odo", "registry", "delete", registryName, "-f") + }) + + It("Should successfully update the registry", func() { + helper.CmdShouldPass("odo", "registry", "add", registryName, addRegistryURL) + helper.CmdShouldPass("odo", "registry", "update", registryName, updateRegistryURL, "-f") + output := helper.CmdShouldPass("odo", "registry", "list") + helper.MatchAllInOutput(output, []string{registryName, updateRegistryURL}) + helper.CmdShouldPass("odo", "registry", "delete", registryName, "-f") + }) + + It("Should successfully delete the registry", func() { + helper.CmdShouldPass("odo", "registry", "add", registryName, addRegistryURL) + helper.CmdShouldPass("odo", "registry", "delete", registryName, "-f") + helper.CmdShouldFail("odo", "create", "maven", "--registry", registryName) + }) + }) +})