Dynamic registry support (redhat-developer#2940)

* Draft PR for dynamic registry support Signed-off-by: jingfu wang <jingfu.j.wang@ibm.com> * Handle migration Signed-off-by: jingfu wang <jingfu.j.wang@ibm.com> * Handle migration with experimental Signed-off-by: jingfu wang <jingfu.j.wang@ibm.com> * Improve error handling Signed-off-by: jingfu wang <jingfu.j.wang@ibm.com> * Add unit tests Signed-off-by: jingfu wang <jingfu.j.wang@ibm.com> * Fix unit tests Signed-off-by: jingfu wang <jingfu.j.wang@ibm.com> * Fix unit tests Signed-off-by: jingfu wang <jingfu.j.wang@ibm.com> * Add integration tests Signed-off-by: jingfu wang <jingfu.j.wang@ibm.com> * Update "odo delete" and display registry name Signed-off-by: jingfu wang <jingfu.j.wang@ibm.com> * Add confirmation dialog for update and delete Signed-off-by: jingfu wang <jingfu.j.wang@ibm.com> * Fix catalog test Signed-off-by: jingfu wang <jingfu.j.wang@ibm.com> * Update help page and delete functions Signed-off-by: jingfu wang <jingfu.j.wang@ibm.com> * Help page cleanup Signed-off-by: jingfu wang <jingfu.j.wang@ibm.com> * update confirmation page Signed-off-by: jingfu wang <jingfu.j.wang@ibm.com> * Add URL validation Signed-off-by: jingfu wang <jingfu.j.wang@ibm.com> * Fix unit test Signed-off-by: jingfu wang <jingfu.j.wang@ibm.com> * Use built-in library for URL parsing Signed-off-by: jingfu wang <jingfu.j.wang@ibm.com> * Update message Signed-off-by: jingfu wang <jingfu.j.wang@ibm.com> * Update help page and use const default registry Signed-off-by: jingfu wang <jingfu.j.wang@ibm.com> * Update registry URL Signed-off-by: jingfu wang <jingfu.j.wang@ibm.com> * Update help page Signed-off-by: jingfu wang <jingfu.j.wang@ibm.com> * Update registry tests Signed-off-by: jingfu wang <jingfu.j.wang@ibm.com> * Add github registry example Signed-off-by: jingfu wang <jingfu.j.wang@ibm.com> * Update template Signed-off-by: jingfu wang <jingfu.j.wang@ibm.com> * Fix typo Signed-off-by: jingfu wang <jingfu.j.wang@ibm.com> * Update k8s packages Signed-off-by: jingfu wang <jingfu.j.wang@ibm.com> * Fix registry test Signed-off-by: jingfu wang <jingfu.j.wang@ibm.com>
metacosm · May 9, 2020 · bfa9922 · bfa9922
1 parent bf6bf02
commit bfa9922
Show file tree

Hide file tree

Showing 20 changed files with 1,004 additions and 26 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -130,7 +130,7 @@ jobs:
       # scenario of docker devfile url testing needs only Kube config file. So the test has been
       # added here just to make sure docker devfile url command test gets a proper kube config file.
       # without creating a separate OpenShift cluster.
-      name: "devfile catalog, create, push, delete and docker devfile url command integration tests"
+      name: "devfile catalog, create, push, delete, registry and docker devfile url command integration tests"
       script:
         - ./scripts/oc-cluster.sh
         - make bin
@@ -143,6 +143,7 @@ jobs:
         - travis_wait make test-cmd-devfile-push
         - travis_wait make test-cmd-devfile-watch
         - travis_wait make test-cmd-devfile-delete
+        - travis_wait make test-cmd-devfile-registry
         - odo logout
 
     - <<: *base-test

diff --git a/Makefile b/Makefile
@@ -217,6 +217,11 @@ test-cmd-devfile-watch:
 .PHONY: test-cmd-devfile-delete
 test-cmd-devfile-delete:
 	ginkgo $(GINKGO_FLAGS) -focus="odo devfile delete command tests" tests/integration/devfile/
+
+# Run odo devfile registry command tests
+.PHONY: test-cmd-devfile-registry
+test-cmd-devfile-registry:
+	ginkgo $(GINKGO_FLAGS) -focus="odo devfile registry command tests" tests/integration/devfile/
 
 # Run odo storage command tests
 .PHONY: test-cmd-storage

diff --git a/pkg/catalog/catalog.go b/pkg/catalog/catalog.go
@@ -7,18 +7,42 @@ import (
 	"strings"
 
 	imagev1 "github.com/openshift/api/image/v1"
+	"github.com/openshift/odo/pkg/log"
 	"github.com/openshift/odo/pkg/occlient"
+	"github.com/openshift/odo/pkg/preference"
 	"github.com/openshift/odo/pkg/util"
 	"github.com/pkg/errors"
 	"gopkg.in/yaml.v2"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/klog"
 )
 
-// DevfileRegistries contains the links of all devfile registries
-var DevfileRegistries = []string{
-	"https://raw.githubusercontent.com/elsony/devfile-registry/master",
-	"https://che-devfile-registry.openshift.io/",
+// GetDevfileRegistries gets devfile registries from preference file,
+// if registry name is specified return the specific registry, otherwise return all registries
+func GetDevfileRegistries(registryName string) (map[string]string, error) {
+	devfileRegistries := make(map[string]string)
+
+	cfg, err := preference.New()
+	if err != nil {
+		return nil, err
+	}
+
+	if cfg.OdoSettings.RegistryList != nil {
+		for _, registry := range *cfg.OdoSettings.RegistryList {
+			if len(registryName) != 0 {
+				if registryName == registry.Name {
+					devfileRegistries[registry.Name] = registry.URL
+					return devfileRegistries, nil
+				}
+			} else {
+				devfileRegistries[registry.Name] = registry.URL
+			}
+		}
+	} else {
+		return nil, nil
+	}
+
+	return devfileRegistries, nil
 }
 
 // GetDevfileIndex loads the devfile registry index.json
@@ -103,16 +127,26 @@ func IsDevfileComponentSupported(devfile Devfile) bool {
 }
 
 // ListDevfileComponents lists all the available devfile components
-func ListDevfileComponents() (DevfileComponentTypeList, error) {
+func ListDevfileComponents(registryName string) (DevfileComponentTypeList, error) {
 	var catalogDevfileList DevfileComponentTypeList
-	catalogDevfileList.DevfileRegistries = DevfileRegistries
+	var err error
+
+	// Get devfile registries
+	catalogDevfileList.DevfileRegistries, err = GetDevfileRegistries(registryName)
+	if err != nil {
+		return catalogDevfileList, err
+	}
+	if catalogDevfileList.DevfileRegistries == nil {
+		return catalogDevfileList, nil
+	}
 
-	for _, devfileRegistry := range DevfileRegistries {
+	for registryName, registryURL := range catalogDevfileList.DevfileRegistries {
 		// Load the devfile registry index.json
-		devfileIndexLink := devfileRegistry + "/devfiles/index.json"
+		devfileIndexLink := registryURL + "/devfiles/index.json"
 		devfileIndex, err := GetDevfileIndex(devfileIndexLink)
 		if err != nil {
-			return DevfileComponentTypeList{}, err
+			log.Warningf("Registry %s is not set up properly with error: %v", registryName, err)
+			break
 		}
 
 		// 1. Load devfiles that indexed in devfile registry index.json
@@ -122,14 +156,15 @@ func ListDevfileComponents() (DevfileComponentTypeList, error) {
 			devfileIndexEntryLink := devfileIndexEntry.Links.Link
 
 			// Load the devfile
-			devfileLink := devfileRegistry + devfileIndexEntryLink
-			// TODO: We send http get resquest in this function mutiple times
+			devfileLink := registryURL + devfileIndexEntryLink
+			// TODO: We send http get resquest in this function multiple times
 			// since devfile registry uses different links to host different devfiles,
 			// this can reduce the performance especially when we load devfiles from
 			// big registry. We may need to rethink and optimize this in the future
 			devfile, err := GetDevfile(devfileLink)
 			if err != nil {
-				return DevfileComponentTypeList{}, err
+				log.Warningf("Registry %s is not set up properly with error: %v", registryName, err)
+				break
 			}
 
 			// Populate devfile component with devfile data and form devfile component list
@@ -139,7 +174,10 @@ func ListDevfileComponents() (DevfileComponentTypeList, error) {
 				Description: devfileIndexEntry.Description,
 				Link:        devfileIndexEntryLink,
 				Support:     IsDevfileComponentSupported(devfile),
-				Registry:    devfileRegistry,
+				Registry: Registry{
+					Name: registryName,
+					URL:  registryURL,
+				},
 			}
 
 			catalogDevfileList.Items = append(catalogDevfileList.Items, catalogDevfile)

diff --git a/pkg/catalog/catalog_test.go b/pkg/catalog/catalog_test.go
@@ -1,13 +1,16 @@
 package catalog
 
 import (
+	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"reflect"
 	"testing"
 
 	imagev1 "github.com/openshift/api/image/v1"
 	"github.com/openshift/odo/pkg/occlient"
+	"github.com/openshift/odo/pkg/preference"
 	"github.com/openshift/odo/pkg/testingutil"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -168,6 +171,67 @@ func TestSliceSupportedTags(t *testing.T) {
 	}
 }
 
+func TestGetDevfileRegistries(t *testing.T) {
+	tempConfigFile, err := ioutil.TempFile("", "odoconfig")
+	if err != nil {
+		t.Fatal("Fail to create temporary config file")
+	}
+	defer os.Remove(tempConfigFile.Name())
+	defer tempConfigFile.Close()
+	_, err = tempConfigFile.Write([]byte(
+		`kind: Preference
+apiversion: odo.openshift.io/v1alpha1
+OdoSettings:
+  Experimental: true
+  RegistryList:
+  - Name: CheDevfileRegistry
+    URL: https://che-devfile-registry.openshift.io/
+  - Name: DefaultDevfileRegistry
+    URL: https://raw.githubusercontent.com/elsony/devfile-registry/master`,
+	))
+	if err != nil {
+		t.Error(err)
+	}
+
+	os.Setenv(preference.GlobalConfigEnvName, tempConfigFile.Name())
+	defer os.Unsetenv(preference.GlobalConfigEnvName)
+
+	tests := []struct {
+		name         string
+		registryName string
+		want         map[string]string
+	}{
+		{
+			name:         "Case 1: Test get all devfile registries",
+			registryName: "",
+			want: map[string]string{
+				"CheDevfileRegistry":     "https://che-devfile-registry.openshift.io/",
+				"DefaultDevfileRegistry": "https://raw.githubusercontent.com/elsony/devfile-registry/master",
+			},
+		},
+		{
+			name:         "Case 2: Test get specific devfile registry",
+			registryName: "CheDevfileRegistry",
+			want: map[string]string{
+				"CheDevfileRegistry": "https://che-devfile-registry.openshift.io/",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := GetDevfileRegistries(tt.registryName)
+			if err != nil {
+				t.Errorf("Error message is %v", err)
+			}
+
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("Got: %v, want: %v", got, tt.want)
+			}
+		})
+	}
+}
+
 func TestGetDevfileIndex(t *testing.T) {
 	// Start a local HTTP server
 	server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {

diff --git a/pkg/catalog/types.go b/pkg/catalog/types.go
@@ -12,14 +12,20 @@ type ComponentType struct {
 	Spec              ComponentSpec `json:"spec,omitempty"`
 }
 
+// Registry is the main struct of devfile registry
+type Registry struct {
+	Name string
+	URL  string
+}
+
 // DevfileComponentType is the main struct for devfile catalog components
 type DevfileComponentType struct {
 	Name        string
 	DisplayName string
 	Description string
 	Link        string
 	Support     bool
-	Registry    string
+	Registry    Registry
 }
 
 // DevfileIndexEntry is the main struct of index.json from devfile registry
@@ -66,7 +72,7 @@ type ComponentTypeList struct {
 
 // DevfileComponentTypeList lists all the DevfileComponentType's
 type DevfileComponentTypeList struct {
-	DevfileRegistries []string
+	DevfileRegistries map[string]string
 	Items             []DevfileComponentType
 }
 

diff --git a/pkg/odo/cli/catalog/list/components.go b/pkg/odo/cli/catalog/list/components.go
@@ -57,10 +57,14 @@ func (o *ListComponentsOptions) Complete(name string, cmd *cobra.Command, args [
 	}
 
 	if experimental.IsExperimentalModeEnabled() {
-		o.catalogDevfileList, err = catalog.ListDevfileComponents()
+		o.catalogDevfileList, err = catalog.ListDevfileComponents("")
 		if err != nil {
 			return err
 		}
+
+		if o.catalogDevfileList.DevfileRegistries == nil {
+			log.Warning("Please run 'odo registry add <registry name> <registry URL>' to add registry for listing devfile components\n")
+		}
 	}
 
 	return
@@ -130,7 +134,7 @@ func (o *ListComponentsOptions) Run() (err error) {
 
 		if len(supDevfileCatalogList) != 0 || (o.listAllDevfileComponents && len(unsupDevfileCatalogList) != 0) {
 			fmt.Fprintln(w, "Odo Devfile Components:")
-			fmt.Fprintln(w, "NAME", "\t", "DESCRIPTION", "\t", "SUPPORTED")
+			fmt.Fprintln(w, "NAME", "\t", "DESCRIPTION", "\t", "REGISTRY", "\t", "SUPPORTED")
 
 			if len(supDevfileCatalogList) != 0 {
 				supported = "YES"
@@ -192,6 +196,6 @@ func (o *ListComponentsOptions) printCatalogList(w io.Writer, catalogList []cata
 
 func (o *ListComponentsOptions) printDevfileCatalogList(w io.Writer, catalogDevfileList []catalog.DevfileComponentType, supported string) {
 	for _, devfileComponent := range catalogDevfileList {
-		fmt.Fprintln(w, devfileComponent.Name, "\t", devfileComponent.Description, "\t", supported)
+		fmt.Fprintln(w, devfileComponent.Name, "\t", devfileComponent.Description, "\t", devfileComponent.Registry.Name, "\t", supported)
 	}
 }
diff --git a/pkg/odo/cli/cli.go b/pkg/odo/cli/cli.go
@@ -16,13 +16,15 @@ import (
 	"github.com/openshift/odo/pkg/odo/cli/plugins"
 	"github.com/openshift/odo/pkg/odo/cli/preference"
 	"github.com/openshift/odo/pkg/odo/cli/project"
+	"github.com/openshift/odo/pkg/odo/cli/registry"
 	"github.com/openshift/odo/pkg/odo/cli/service"
 	"github.com/openshift/odo/pkg/odo/cli/storage"
 	"github.com/openshift/odo/pkg/odo/cli/url"
 	"github.com/openshift/odo/pkg/odo/cli/utils"
 	"github.com/openshift/odo/pkg/odo/cli/version"
 	"github.com/openshift/odo/pkg/odo/util"
 	odoutil "github.com/openshift/odo/pkg/odo/util"
+	"github.com/openshift/odo/pkg/odo/util/experimental"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
@@ -197,6 +199,12 @@ func odoRootCmd(name, fullName string) *cobra.Command {
 		debug.NewCmdDebug(debug.RecommendedCommandName, util.GetFullName(fullName, debug.RecommendedCommandName)),
 	)
 
+	if experimental.IsExperimentalModeEnabled() {
+		rootCmd.AddCommand(
+			registry.NewCmdRegistry(registry.RecommendedCommandName, util.GetFullName(fullName, registry.RecommendedCommandName)),
+		)
+	}
+
 	odoutil.VisitCommands(rootCmd, reconfigureCmdWithSubcmd)
 
 	return rootCmd

diff --git a/pkg/odo/cli/component/create.go b/pkg/odo/cli/component/create.go
@@ -63,7 +63,7 @@ type DevfileMetadata struct {
 	componentNamespace string
 	devfileSupport     bool
 	devfileLink        string
-	devfileRegistry    string
+	devfileRegistry    catalog.Registry
 	downloadSource     bool
 }
 
@@ -329,10 +329,13 @@ func (co *CreateOptions) Complete(name string, cmd *cobra.Command, args []string
 			co.CommonPushOptions.componentContext = co.componentContext
 		}
 
-		catalogDevfileList, err := catalog.ListDevfileComponents()
+		catalogDevfileList, err := catalog.ListDevfileComponents(co.devfileMetadata.devfileRegistry.Name)
 		if err != nil {
 			return err
 		}
+		if catalogDevfileList.DevfileRegistries == nil {
+			log.Warning("Please run `odo registry add <registry name> <registry URL>` to add a registry then create a devfile components\n")
+		}
 
 		var componentType string
 		var componentName string
@@ -437,18 +440,22 @@ func (co *CreateOptions) Complete(name string, cmd *cobra.Command, args []string
 			}
 		}
 
+		registrySpinner := log.Spinnerf("Creating a devfile component from registry: %s", co.devfileMetadata.devfileRegistry.Name)
+
 		if co.devfileMetadata.devfileSupport {
 			err = co.InitEnvInfoFromContext()
 			if err != nil {
 				return err
 			}
 
 			spinner.End(true)
+			registrySpinner.End(true)
 			return nil
 		}
 
 		spinner.End(false)
-		log.Italic("\nPlease run 'odo catalog list components' for a list of supported devfile component types")
+		registrySpinner.End(false)
+		log.Italic("\nPlease run `odo catalog list components` for a list of supported devfile component types")
 	}
 
 	if len(args) == 0 || !cmd.HasFlags() {
@@ -773,7 +780,7 @@ func (co *CreateOptions) Run() (err error) {
 		// Download devfile.yaml file and create env.yaml file
 		if co.devfileMetadata.devfileSupport {
 			if !util.CheckPathExists(DevfilePath) {
-				err := util.DownloadFile(co.devfileMetadata.devfileRegistry+co.devfileMetadata.devfileLink, DevfilePath)
+				err := util.DownloadFile(co.devfileMetadata.devfileRegistry.URL+co.devfileMetadata.devfileLink, DevfilePath)
 				if err != nil {
 					return errors.Wrap(err, "Faile to download devfile.yaml for devfile component")
 				}
@@ -899,6 +906,7 @@ func NewCmdCreate(name, fullName string) *cobra.Command {
 	componentCreateCmd.Flags().StringSliceVar(&co.componentEnvVars, "env", []string{}, "Environmental variables for the component. For example --env VariableName=Value")
 
 	if experimental.IsExperimentalModeEnabled() {
+		componentCreateCmd.Flags().StringVar(&co.devfileMetadata.devfileRegistry.Name, "registry", "", "Create devfile component from specific registry")
 		componentCreateCmd.Flags().BoolVar(&co.devfileMetadata.downloadSource, "downloadSource", false, "Download sample project from devfile. (ex. odo component create <component_type> [component_name] --downloadSource")
 	}