Add nightly check for Product and Community registries (#15)

* Add binaries to every new release (#237) * Add release yaml to workflows Signed-off-by: thepetk <thepetk@gmail.com> * Remove autogeneration of release notes Signed-off-by: thepetk <thepetk@gmail.com> * Update readme Signed-off-by: thepetk <thepetk@gmail.com> * Update Readme.md Signed-off-by: thepetk <thepetk@gmail.com> --------- Signed-off-by: thepetk <thepetk@gmail.com> * flattening go dir (#1) Signed-off-by: Michael Hoang <mhoang@redhat.com> Signed-off-by: thepetk <thepetk@gmail.com> * Update realease.yaml (#2) Signed-off-by: thepetk <thepetk@gmail.com> * adding proposal for dockerfile components (#3) Signed-off-by: Michael Hoang <mhoang@redhat.com> Signed-off-by: thepetk <thepetk@gmail.com> * Merge all test resources (#6) * Update test paths Signed-off-by: thepetk <thepetk@gmail.com> * Update angular test resource Signed-off-by: thepetk <thepetk@gmail.com> * Remove projectAngularjs Signed-off-by: thepetk <thepetk@gmail.com> * Update containerfile test Signed-off-by: thepetk <thepetk@gmail.com> * Update django test resources Signed-off-by: thepetk <thepetk@gmail.com> * Update docker compose test resources Signed-off-by: thepetk <thepetk@gmail.com> * Update tests for docker compose with ports Signed-off-by: thepetk <thepetk@gmail.com> * Update test project dockerfile Signed-off-by: thepetk <thepetk@gmail.com> * Update express js port tests Signed-off-by: thepetk <thepetk@gmail.com> * Update flask port tests Signed-off-by: thepetk <thepetk@gmail.com> * Update golang test resources Signed-off-by: thepetk <thepetk@gmail.com> * Update jboss test resources Signed-off-by: thepetk <thepetk@gmail.com> * Update laravel test resources Signed-off-by: thepetk <thepetk@gmail.com> * Update test micronaut resources Signed-off-by: thepetk <thepetk@gmail.com> * Update container docker file nested tests Signed-off-by: thepetk <thepetk@gmail.com> * Update nuxt and next js tests Signed-off-by: thepetk <thepetk@gmail.com> * Update port test for quarkus Signed-off-by: thepetk <thepetk@gmail.com> * Update reactjs tests Signed-off-by: thepetk <thepetk@gmail.com> * Remove port test project quarkus Signed-off-by: thepetk <thepetk@gmail.com> * Update rest of port tests Signed-off-by: thepetk <thepetk@gmail.com> * Fix issue with ip host go format Signed-off-by: thepetk <thepetk@gmail.com> * Update test resources Signed-off-by: thepetk <thepetk@gmail.com> * Finalize new component detection format Signed-off-by: thepetk <thepetk@gmail.com> --------- Signed-off-by: thepetk <thepetk@gmail.com> * Add min max cli args (#5) * Create dependabot.yml Signed-off-by: thepetk <thepetk@gmail.com> * Update dependabot.yml Signed-off-by: thepetk <thepetk@gmail.com> * Update realease.yaml (#2) Signed-off-by: thepetk <thepetk@gmail.com> * adding proposal for dockerfile components (#3) Signed-off-by: Michael Hoang <mhoang@redhat.com> Signed-off-by: thepetk <thepetk@gmail.com> * Merge all test resources (#6) * Update test paths Signed-off-by: thepetk <thepetk@gmail.com> * Update angular test resource Signed-off-by: thepetk <thepetk@gmail.com> * Remove projectAngularjs Signed-off-by: thepetk <thepetk@gmail.com> * Update containerfile test Signed-off-by: thepetk <thepetk@gmail.com> * Update django test resources Signed-off-by: thepetk <thepetk@gmail.com> * Update docker compose test resources Signed-off-by: thepetk <thepetk@gmail.com> * Update tests for docker compose with ports Signed-off-by: thepetk <thepetk@gmail.com> * Update test project dockerfile Signed-off-by: thepetk <thepetk@gmail.com> * Update express js port tests Signed-off-by: thepetk <thepetk@gmail.com> * Update flask port tests Signed-off-by: thepetk <thepetk@gmail.com> * Update golang test resources Signed-off-by: thepetk <thepetk@gmail.com> * Update jboss test resources Signed-off-by: thepetk <thepetk@gmail.com> * Update laravel test resources Signed-off-by: thepetk <thepetk@gmail.com> * Update test micronaut resources Signed-off-by: thepetk <thepetk@gmail.com> * Update container docker file nested tests Signed-off-by: thepetk <thepetk@gmail.com> * Update nuxt and next js tests Signed-off-by: thepetk <thepetk@gmail.com> * Update port test for quarkus Signed-off-by: thepetk <thepetk@gmail.com> * Update reactjs tests Signed-off-by: thepetk <thepetk@gmail.com> * Remove port test project quarkus Signed-off-by: thepetk <thepetk@gmail.com> * Update rest of port tests Signed-off-by: thepetk <thepetk@gmail.com> * Fix issue with ip host go format Signed-off-by: thepetk <thepetk@gmail.com> * Update test resources Signed-off-by: thepetk <thepetk@gmail.com> * Finalize new component detection format Signed-off-by: thepetk <thepetk@gmail.com> --------- Signed-off-by: thepetk <thepetk@gmail.com> * Run tidy Signed-off-by: thepetk <thepetk@gmail.com> * Update devfile_recognizer and models Signed-off-by: thepetk <thepetk@gmail.com> * Update cli Signed-off-by: thepetk <thepetk@gmail.com> * Update docs Signed-off-by: thepetk <thepetk@gmail.com> * Add test cases for versions cli args Signed-off-by: thepetk <thepetk@gmail.com> * Fix sec alert Signed-off-by: thepetk <thepetk@gmail.com> * Fix typo Signed-off-by: thepetk <thepetk@gmail.com> --------- Signed-off-by: thepetk <thepetk@gmail.com> Signed-off-by: Michael Hoang <mhoang@redhat.com> Co-authored-by: Michael Hoang <35011707+mike-hoang@users.noreply.github.com> Signed-off-by: thepetk <thepetk@gmail.com> * Remove dependabot (#10) Signed-off-by: thepetk@gmail.com Signed-off-by: thepetk@gmail.com Signed-off-by: thepetk <thepetk@gmail.com> * Minor update on devfiles versioning (#11) * Remove dependabot Signed-off-by: thepetk@gmail.com Signed-off-by: thepetk <thepetk@gmail.com> * Add versions to alizer devfile response Signed-off-by: thepetk <thepetk@gmail.com> * Update readme.md Signed-off-by: thepetk <thepetk@gmail.com> * Update naming and devfile models in the proposal Signed-off-by: thepetk <thepetk@gmail.com> * Update code naming Signed-off-by: thepetk <thepetk@gmail.com> * Update tests after renaming Signed-off-by: thepetk <thepetk@gmail.com> --------- Signed-off-by: thepetk@gmail.com Signed-off-by: thepetk <thepetk@gmail.com> * fixing gosec alerts (#12) Signed-off-by: Michael Hoang <mhoang@redhat.com> Signed-off-by: thepetk <thepetk@gmail.com> * Add test coverage check to CI.yaml (#13) * Add test coverage workflow Signed-off-by: thepetk <thepetk@gmail.com> * Update ci.yaml Signed-off-by: thepetk <thepetk@gmail.com> * Add separate check for code coverage Signed-off-by: thepetk <thepetk@gmail.com> * Move code report in ci file Signed-off-by: thepetk <thepetk@gmail.com> * Add .codecov.yaml Signed-off-by: thepetk <thepetk@gmail.com> * Update workflow Signed-off-by: thepetk <thepetk@gmail.com> * Bump up setup-go Signed-off-by:thepetk <thepetk@gmail.com> --------- Signed-off-by: thepetk <thepetk@gmail.com> * adding support for dockerfile components (#14) Signed-off-by: Michael Hoang <mhoang@redhat.com> Signed-off-by: thepetk <thepetk@gmail.com> * Add test coverage workflow Signed-off-by: thepetk <thepetk@gmail.com> * Update ci.yaml Signed-off-by: thepetk <thepetk@gmail.com> * Add separate check for code coverage Signed-off-by: thepetk <thepetk@gmail.com> * Move code report in ci file Signed-off-by: thepetk <thepetk@gmail.com> * Update workflow Signed-off-by: thepetk <thepetk@gmail.com> * Bump up setup-go Signed-off-by:thepetk <thepetk@gmail.com> Signed-off-by: thepetk <thepetk@gmail.com> * Make DownloadDevFileTypesFromRegistry public Signed-off-by: thepetk <thepetk@gmail.com> * Add devfile.yaml schema Signed-off-by: thepetk <thepetk@gmail.com> * Add go script for generating registry entries json Signed-off-by: thepetk <thepetk@gmail.com> * Implement nightly run script and workflow Signed-off-by: thepetk <thepetk@gmail.com> * Update go mod Signed-off-by: thepetk <thepetk@gmail.com> * Add new schedule to registry check signed-off-by: thepetk <thepetk@gmail.com> Signed-off-by: thepetk <thepetk@gmail.com> * Update workflow name Signed-off-by: thepetk <thepetk@gmail.com> * Update go mod Signed-off-by: thepetk <thepetk@gmail.com> * Update funcs in order to be mockable Signed-off-by: thepetk <thepetk@gmail.com> * Move devfile_recognizer_test.go to recognizer dir Signed-off-by: thepetk <thepetk@gmail.com> * Update docstring of script Signed-off-by: thepetk <thepetk@gmail.com> * Add tests for check_registry.go Signed-off-by: thepetk <thepetk@gmail.com> * Fix test paths Signed-off-by: thepetk <thepetk@gmail.com> * Remove unecessary logging Signed-off-by: thepetk <thepetk@gmail.com> * Remove binary Signed-off-by: thepetk <thepetk@gmail.com> * Use make build instead of go command Signed-off-by: thepetk <thepetk@gmail.com> * Further fixes on the workflow Signed-off-by: thepetk <thepetk@gmail.com> --------- Signed-off-by: thepetk <thepetk@gmail.com> Signed-off-by: Michael Hoang <mhoang@redhat.com> Signed-off-by: thepetk@gmail.com Co-authored-by: Michael Hoang <35011707+mike-hoang@users.noreply.github.com>
devfile · Aug 1, 2023 · bef6b03 · bef6b03
1 parent 7f521b2
commit bef6b03
Show file tree

Hide file tree

Showing 8 changed files with 522 additions and 12 deletions.
diff --git a/.github/workflows/check_registry.yaml b/.github/workflows/check_registry.yaml
@@ -0,0 +1,35 @@
+on:
+  schedule:
+  - cron: "0 3 * * *"
+
+name: Product & Community Registries Check
+
+defaults:
+  run:
+    shell: bash
+
+permissions:
+  contents: write
+
+jobs:
+  check-registry:
+    name: Check registry entries
+    runs-on: 'ubuntu-latest'
+    outputs:
+      matrix: ${{ steps.get-stacks.outputs.matrix }}
+    steps:
+      - name: Checkout code
+        id: checkout-code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Setup go
+        id: setup-go
+        uses: actions/setup-go@v4
+        with:
+          go-version: '1.19.5'
+      - name: Run registries check
+        id: build
+        run: |
+          make build
+          bash test/check_registry/check_registry.sh
diff --git a/go.mod b/go.mod
@@ -14,6 +14,7 @@ require (
 	github.com/stretchr/testify v1.8.1
 	go.uber.org/zap v1.24.0
 	golang.org/x/mod v0.11.0
+	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
 	sigs.k8s.io/controller-runtime v0.15.0
 )
@@ -55,7 +56,6 @@ require (
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 	k8s.io/apimachinery v0.27.3 // indirect
 	k8s.io/klog/v2 v2.100.1 // indirect
 	k8s.io/utils v0.0.0-20230505201702-9f6742963106 // indirect

diff --git a/pkg/apis/recognizer/devfile_recognizer.go b/pkg/apis/recognizer/devfile_recognizer.go
@@ -109,7 +109,7 @@ func MatchDevfiles(path string, url string, filter model.DevfileFilter) ([]model
 	alizerLogger := utils.GetOrCreateLogger()
 	alizerLogger.V(0).Info("Starting devfile matching")
 	alizerLogger.V(1).Info(fmt.Sprintf("Downloading devfiles from registry %s", url))
-	devFileTypesFromRegistry, err := downloadDevFileTypesFromRegistry(url, filter)
+	devFileTypesFromRegistry, err := DownloadDevFileTypesFromRegistry(url, filter)
 	if err != nil {
 		return []model.DevFileType{}, err
 	}
@@ -121,15 +121,16 @@ func SelectDevFilesFromRegistry(path string, url string) ([]model.DevFileType, e
 	alizerLogger := utils.GetOrCreateLogger()
 	alizerLogger.V(0).Info("Starting devfile matching")
 	alizerLogger.V(1).Info(fmt.Sprintf("Downloading devfiles from registry %s", url))
-	devFileTypesFromRegistry, err := downloadDevFileTypesFromRegistry(url, model.DevfileFilter{MinSchemaVersion: "", MaxSchemaVersion: ""})
+	devFileTypesFromRegistry, err := DownloadDevFileTypesFromRegistry(url, model.DevfileFilter{MinSchemaVersion: "", MaxSchemaVersion: ""})
 	if err != nil {
 		return []model.DevFileType{}, err
 	}
 
 	return selectDevfiles(path, devFileTypesFromRegistry)
 }
 
-func selectDevfiles(path string, devFileTypesFromRegistry []model.DevFileType) ([]model.DevFileType, error) {
+// selectDevfiles is exposed as global var in the purpose of mocking tests
+var selectDevfiles = func(path string, devFileTypesFromRegistry []model.DevFileType) ([]model.DevFileType, error) {
 	indexes, err := SelectDevFilesFromTypes(path, devFileTypesFromRegistry)
 	if err != nil {
 		return []model.DevFileType{}, err
@@ -145,7 +146,7 @@ func selectDevfiles(path string, devFileTypesFromRegistry []model.DevFileType) (
 }
 
 func SelectDevFileFromRegistry(path string, url string) (model.DevFileType, error) {
-	devFileTypes, err := downloadDevFileTypesFromRegistry(url, model.DevfileFilter{MinSchemaVersion: "", MaxSchemaVersion: ""})
+	devFileTypes, err := DownloadDevFileTypesFromRegistry(url, model.DevfileFilter{MinSchemaVersion: "", MaxSchemaVersion: ""})
 	if err != nil {
 		return model.DevFileType{}, err
 	}
@@ -203,7 +204,8 @@ func GetUrlWithVersions(url, minSchemaVersion, maxSchemaVersion string) (string,
 	}
 }
 
-func downloadDevFileTypesFromRegistry(url string, filter model.DevfileFilter) ([]model.DevFileType, error) {
+// DownloadDevFileTypesFromRegistry is exposed as a global variable for the purpose of running mock tests
+var DownloadDevFileTypesFromRegistry = func(url string, filter model.DevfileFilter) ([]model.DevFileType, error) {
 	url = adaptUrl(url)
 	tmpUrl := appendIndexPath(url)
 	url, err := GetUrlWithVersions(tmpUrl, filter.MinSchemaVersion, filter.MaxSchemaVersion)

diff --git a/test/apis/devfile_recognizer_test.go → ...pis/recognizer/devfile_recognizer_test.go b/test/apis/devfile_recognizer_test.go → ...pis/recognizer/devfile_recognizer_test.go
@@ -12,10 +12,12 @@ package recognizer
  ******************************************************************************/
 import (
 	"fmt"
+	"path/filepath"
+	"reflect"
+	"runtime"
 	"testing"
 
 	"github.com/devfile/alizer/pkg/apis/model"
-	"github.com/devfile/alizer/pkg/apis/recognizer"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -205,7 +207,7 @@ func TestGetUrlWithVersions(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			result, err := recognizer.GetUrlWithVersions(tt.testUrl, tt.minSchemaVersion, tt.maxSchemaVersion)
+			result, err := GetUrlWithVersions(tt.testUrl, tt.minSchemaVersion, tt.maxSchemaVersion)
 			if err != nil {
 				assert.EqualValues(t, tt.expectedError, err)
 			} else {
@@ -215,6 +217,55 @@ func TestGetUrlWithVersions(t *testing.T) {
 	}
 }
 
+func TestMatchDevfiles(t *testing.T) {
+	tests := []struct {
+		name                string
+		filter              model.DevfileFilter
+		mockFunc            func()
+		url                 string
+		path                string
+		expectedDevfileType model.DevFileType
+		expectingErr        bool
+	}{
+		{
+			name:   "Match devfile success",
+			filter: model.DevfileFilter{},
+			path:   "some-path",
+			url:    "some-url",
+			expectedDevfileType: model.DevFileType{
+				Name:        "mocked-stack",
+				Language:    "python",
+				ProjectType: "python",
+				Tags:        []string{"python"},
+				Versions:    []model.Version{},
+			},
+			expectingErr: false,
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(tt *testing.T) {
+			// mock DownloadDevFileTypesFromRegistry
+			DownloadDevFileTypesFromRegistry = func(url string, filter model.DevfileFilter) ([]model.DevFileType, error) {
+				return []model.DevFileType{tc.expectedDevfileType}, nil
+			}
+			selectDevfiles = func(path string, devFileTypesFromRegistry []model.DevFileType) ([]model.DevFileType, error) {
+				return []model.DevFileType{tc.expectedDevfileType}, nil
+			}
+			devfileType, err := MatchDevfiles("test-path", "some-url", model.DevfileFilter{})
+
+			errExist := err != nil
+			if tc.expectingErr != errExist {
+				tt.Errorf("Error expectation not met, want %v, got %v", tc.expectingErr, errExist)
+			}
+
+			if !reflect.DeepEqual(tc.expectedDevfileType.Name, devfileType[0].Name) {
+				tt.Errorf("Error, user profile expectation not met, want %+v, got %+v", tc.expectedDevfileType, devfileType)
+			}
+		})
+	}
+}
+
 func getExceptedVersionsUrl(url, minSchemaVersion, maxSchemaVersion string, err error) string {
 	if err != nil {
 		return ""
@@ -232,15 +283,15 @@ func getExceptedVersionsUrl(url, minSchemaVersion, maxSchemaVersion string, err
 func detectDevFiles(t *testing.T, projectName string, devFilesName []string) {
 	detectDevFilesFunc := func(devFileTypes []model.DevFileType) ([]int, error) {
 		testingProjectPath := getTestProjectPath(projectName)
-		return recognizer.SelectDevFilesFromTypes(testingProjectPath, devFileTypes)
+		return SelectDevFilesFromTypes(testingProjectPath, devFileTypes)
 	}
 	detectDevFilesInner(t, devFilesName, detectDevFilesFunc)
 }
 
 func detectDevFile(t *testing.T, projectName string, devFilesName []string) {
 	detectDevFilesFunc := func(devFileTypes []model.DevFileType) ([]int, error) {
 		testingProjectPath := getTestProjectPath(projectName)
-		devfileIndex, err := recognizer.SelectDevFileFromTypes(testingProjectPath, devFileTypes)
+		devfileIndex, err := SelectDevFileFromTypes(testingProjectPath, devFileTypes)
 		return []int{devfileIndex}, err
 	}
 	detectDevFilesInner(t, devFilesName, detectDevFilesFunc)
@@ -250,13 +301,13 @@ func detectDevFilesUsingLanguages(t *testing.T, projectName string, languages []
 	if projectName != "" {
 		testingProjectPath := getTestProjectPath(projectName)
 		var err error
-		languages, err = recognizer.Analyze(testingProjectPath)
+		languages, err = Analyze(testingProjectPath)
 		if err != nil {
 			t.Error(err)
 		}
 	}
 	detectDevFileFunc := func(devFileTypes []model.DevFileType) ([]int, error) {
-		return recognizer.SelectDevFilesUsingLanguagesFromTypes(languages, devFileTypes)
+		return SelectDevFilesUsingLanguagesFromTypes(languages, devFileTypes)
 	}
 	detectDevFilesInner(t, devFileName, detectDevFileFunc)
 }
@@ -491,3 +542,9 @@ func getDevFileTypes() []model.DevFileType {
 		},
 	}
 }
+
+func getTestProjectPath(folder string) string {
+	_, b, _, _ := runtime.Caller(0)
+	basepath := filepath.Dir(b)
+	return filepath.Join(basepath, "..", "..", "..", "resources/projects", folder)
+}
diff --git a/pkg/schema/devfile_yaml.go b/pkg/schema/devfile_yaml.go
@@ -0,0 +1,27 @@
+/*******************************************************************************
+ * Copyright (c) 2023 Red Hat, Inc.
+ * Distributed under license by Red Hat, Inc. All rights reserved.
+ * This program is made available under the terms of the
+ * Eclipse Public License v2.0 which accompanies this distribution,
+ * and is available at http://www.eclipse.org/legal/epl-v20.html
+ *
+ * Contributors:
+ * Red Hat, Inc.
+ ******************************************************************************/
+
+package schema
+
+type DevfileYaml struct {
+	StarterProjects []struct {
+		Git struct {
+			CheckoutFrom struct {
+				Remote   string `yaml:"remote"`
+				Revision string `yaml:"revision"`
+			} `yaml:"checkoutFrom"`
+			Remotes struct {
+				Origin string `yaml:"origin"`
+			} `yaml:"remotes"`
+		} `yaml:"git"`
+		SubDir string `yaml:"subDir"`
+	} `yaml:"starterProjects"`
+}