From 608afe9e3867f00e31900481588a9d63f4dd25ac Mon Sep 17 00:00:00 2001 From: Tadayoshi Sato Date: Sat, 20 Aug 2022 11:41:05 +0900 Subject: [PATCH 01/11] chore(util): refactor camel_dependencies.go --- .../camel/v1/camelcatalog_types_support.go | 5 + pkg/util/camel/camel_dependencies.go | 220 ++++++++++-------- pkg/util/camel/camel_runtime_catalog.go | 5 - 3 files changed, 131 insertions(+), 99 deletions(-) diff --git a/pkg/apis/camel/v1/camelcatalog_types_support.go b/pkg/apis/camel/v1/camelcatalog_types_support.go index 0d643bdfb3..777d6bc804 100644 --- a/pkg/apis/camel/v1/camelcatalog_types_support.go +++ b/pkg/apis/camel/v1/camelcatalog_types_support.go @@ -67,6 +67,11 @@ func (c *CamelCatalogSpec) GetRuntimeVersion() string { return c.Runtime.Version } +// GetCamelVersion returns the Camel version the runtime is based on. +func (c *CamelCatalogSpec) GetCamelVersion() string { + return c.Runtime.Metadata["camel.version"] +} + // HasCapability checks if the given capability is present in the catalog. func (c *CamelCatalogSpec) HasCapability(capability string) bool { _, ok := c.Runtime.Capabilities[capability] diff --git a/pkg/util/camel/camel_dependencies.go b/pkg/util/camel/camel_dependencies.go index e6055f5966..509c4021af 100644 --- a/pkg/util/camel/camel_dependencies.go +++ b/pkg/util/camel/camel_dependencies.go @@ -47,113 +47,111 @@ func addDependencies(project *maven.Project, dependencies []string, catalog *Run for _, d := range dependencies { switch { case strings.HasPrefix(d, "bom:"): - gav := strings.TrimPrefix(d, "bom:") - - d, err := maven.ParseGAV(gav) - if err != nil { + if err := addBOM(project, d); err != nil { return err } - - project.DependencyManagement.Dependencies = append(project.DependencyManagement.Dependencies, maven.Dependency{ - GroupID: d.GroupID, - ArtifactID: d.ArtifactID, - Version: d.Version, - Type: "pom", - Scope: "import", - }) case strings.HasPrefix(d, "camel:"): - if catalog != nil && catalog.Runtime.Provider == v1.RuntimeProviderQuarkus { - artifactID := strings.TrimPrefix(d, "camel:") - - if !strings.HasPrefix(artifactID, "camel-") { - artifactID = "camel-quarkus-" + artifactID - } - - project.AddDependencyGAV("org.apache.camel.quarkus", artifactID, "") - } else { - artifactID := strings.TrimPrefix(d, "camel:") - - if !strings.HasPrefix(artifactID, "camel-") { - artifactID = "camel-" + artifactID - } - - project.AddDependencyGAV("org.apache.camel", artifactID, "") - } + addCamelComponent(project, catalog, d) case strings.HasPrefix(d, "camel-k:"): - artifactID := strings.TrimPrefix(d, "camel-k:") - - if !strings.HasPrefix(artifactID, "camel-k-") { - artifactID = "camel-k-" + artifactID - } - - project.AddDependencyGAV("org.apache.camel.k", artifactID, "") + addCamelKComponent(project, d) case strings.HasPrefix(d, "camel-quarkus:"): - artifactID := strings.TrimPrefix(d, "camel-quarkus:") - - if !strings.HasPrefix(artifactID, "camel-quarkus-") { - artifactID = "camel-quarkus-" + artifactID - } - - project.AddDependencyGAV("org.apache.camel.quarkus", artifactID, "") + addCamelQuarkusComponent(project, d) case strings.HasPrefix(d, "mvn:"): - gav := strings.TrimPrefix(d, "mvn:") - - project.AddEncodedDependencyGAV(gav) + addMavenDependency(project, d) case strings.HasPrefix(d, "registry-mvn:"): - mapping := strings.Split(d, "@") - outputFileRelativePath := mapping[1] - gavString := strings.TrimPrefix(mapping[0], "registry-mvn:") - gav, err := maven.ParseGAV(gavString) - if err != nil { + if err := addRegistryMavenDependency(project, d); err != nil { return err } - plugin := getOrCreateBuildPlugin(project, "com.googlecode.maven-download-plugin", "download-maven-plugin", "1.6.7") - exec := maven.Execution{ - ID: fmt.Sprint(len(plugin.Executions)), - Phase: "package", - Goals: []string{ - "artifact", - }, - Configuration: map[string]string{ - "outputDirectory": path.Join("../context", filepath.Dir(outputFileRelativePath)), - "outputFileName": filepath.Base(outputFileRelativePath), - "groupId": gav.GroupID, - "artifactId": gav.ArtifactID, - "version": gav.Version, - "type": gav.Type, - }, - } - plugin.Executions = append(plugin.Executions, exec) default: - if dep := jitpack.ToDependency(d); dep != nil { - project.AddDependency(*dep) - - addRepo := true - for _, repo := range project.Repositories { - if repo.URL == jitpack.RepoURL { - addRepo = false - break - } - } - if addRepo { - project.Repositories = append(project.Repositories, v1.Repository{ - ID: "jitpack.io-" + xid.New().String(), - URL: jitpack.RepoURL, - Releases: v1.RepositoryPolicy{ - Enabled: true, - ChecksumPolicy: "fail", - }, - Snapshots: v1.RepositoryPolicy{ - Enabled: true, - ChecksumPolicy: "fail", - }, - }) - } - } else { - return fmt.Errorf("unknown dependency type: %s", d) + if err := addJitPack(project, d); err != nil { + return err } } } + + return nil +} + +func addBOM(project *maven.Project, dependency string) error { + gav := strings.TrimPrefix(dependency, "bom:") + d, err := maven.ParseGAV(gav) + if err != nil { + return err + } + project.DependencyManagement.Dependencies = append(project.DependencyManagement.Dependencies, + maven.Dependency{ + GroupID: d.GroupID, + ArtifactID: d.ArtifactID, + Version: d.Version, + Type: "pom", + Scope: "import", + }) + + return nil +} + +func addCamelComponent(project *maven.Project, catalog *RuntimeCatalog, dependency string) { + artifactID := strings.TrimPrefix(dependency, "camel:") + if catalog != nil && catalog.Runtime.Provider == v1.RuntimeProviderQuarkus { + if !strings.HasPrefix(artifactID, "camel-") { + artifactID = "camel-quarkus-" + artifactID + } + project.AddDependencyGAV("org.apache.camel.quarkus", artifactID, "") + } else { + if !strings.HasPrefix(artifactID, "camel-") { + artifactID = "camel-" + artifactID + } + project.AddDependencyGAV("org.apache.camel", artifactID, "") + } +} + +func addCamelKComponent(project *maven.Project, dependency string) { + artifactID := strings.TrimPrefix(dependency, "camel-k:") + if !strings.HasPrefix(artifactID, "camel-k-") { + artifactID = "camel-k-" + artifactID + } + project.AddDependencyGAV("org.apache.camel.k", artifactID, "") +} + +func addCamelQuarkusComponent(project *maven.Project, dependency string) { + artifactID := strings.TrimPrefix(dependency, "camel-quarkus:") + if !strings.HasPrefix(artifactID, "camel-quarkus-") { + artifactID = "camel-quarkus-" + artifactID + } + project.AddDependencyGAV("org.apache.camel.quarkus", artifactID, "") +} + +func addMavenDependency(project *maven.Project, dependency string) { + gav := strings.TrimPrefix(dependency, "mvn:") + project.AddEncodedDependencyGAV(gav) +} + +func addRegistryMavenDependency(project *maven.Project, dependency string) error { + mapping := strings.Split(dependency, "@") + outputFileRelativePath := mapping[1] + gavString := strings.TrimPrefix(mapping[0], "registry-mvn:") + gav, err := maven.ParseGAV(gavString) + if err != nil { + return err + } + plugin := getOrCreateBuildPlugin(project, "com.googlecode.maven-download-plugin", "download-maven-plugin", "1.6.7") + exec := maven.Execution{ + ID: fmt.Sprint(len(plugin.Executions)), + Phase: "package", + Goals: []string{ + "artifact", + }, + Configuration: map[string]string{ + "outputDirectory": path.Join("../context", filepath.Dir(outputFileRelativePath)), + "outputFileName": filepath.Base(outputFileRelativePath), + "groupId": gav.GroupID, + "artifactId": gav.ArtifactID, + "version": gav.Version, + "type": gav.Type, + }, + } + plugin.Executions = append(plugin.Executions, exec) + return nil } @@ -170,9 +168,43 @@ func getOrCreateBuildPlugin(project *maven.Project, groupID string, artifactID s Executions: []maven.Execution{}, } project.Build.Plugins = append(project.Build.Plugins, plugin) + return &project.Build.Plugins[len(project.Build.Plugins)-1] } +func addJitPack(project *maven.Project, dependency string) error { + dep := jitpack.ToDependency(dependency) + if dep == nil { + return fmt.Errorf("unknown dependency type: %s", dependency) + } + + project.AddDependency(*dep) + + addRepo := true + for _, repo := range project.Repositories { + if repo.URL == jitpack.RepoURL { + addRepo = false + break + } + } + if addRepo { + project.Repositories = append(project.Repositories, v1.Repository{ + ID: "jitpack.io-" + xid.New().String(), + URL: jitpack.RepoURL, + Releases: v1.RepositoryPolicy{ + Enabled: true, + ChecksumPolicy: "fail", + }, + Snapshots: v1.RepositoryPolicy{ + Enabled: true, + ChecksumPolicy: "fail", + }, + }) + } + + return nil +} + func addDependenciesFromCatalog(project *maven.Project, catalog *RuntimeCatalog) { deps := make([]maven.Dependency, len(project.Dependencies)) copy(deps, project.Dependencies) diff --git a/pkg/util/camel/camel_runtime_catalog.go b/pkg/util/camel/camel_runtime_catalog.go index 5115419110..787370a5fd 100644 --- a/pkg/util/camel/camel_runtime_catalog.go +++ b/pkg/util/camel/camel_runtime_catalog.go @@ -129,11 +129,6 @@ func (c *RuntimeCatalog) GetJavaTypeDependency(camelType string) (string, bool) return javaType, ok } -// GetCamelVersion returns the Camel version the runtime is based on. -func (c *RuntimeCatalog) GetCamelVersion() string { - return c.Runtime.Metadata["camel.version"] -} - // VisitArtifacts --. func (c *RuntimeCatalog) VisitArtifacts(visitor func(string, v1.CamelArtifact) bool) { for id, artifact := range c.Artifacts { From 94fc490b77e8e828fb99b5778912b4515cfbfc9f Mon Sep 17 00:00:00 2001 From: Tadayoshi Sato Date: Sat, 20 Aug 2022 14:49:55 +0900 Subject: [PATCH 02/11] fix(cli): normalise -d completion for bash --- pkg/cmd/completion_bash.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/cmd/completion_bash.go b/pkg/cmd/completion_bash.go index 0c751ef491..1f85aaf8bf 100644 --- a/pkg/cmd/completion_bash.go +++ b/pkg/cmd/completion_bash.go @@ -19,6 +19,7 @@ package cmd import ( "fmt" + "sort" "strings" "github.com/spf13/cobra" @@ -64,7 +65,7 @@ __kamel_dependency_type() { compopt -o nospace ;; *) - local type_list="camel mvn: file:" + local type_list="camel: mvn: file:" COMPREPLY=( $( compgen -W "${type_list}" -- "$cur") ) compopt -o nospace esac @@ -297,9 +298,13 @@ func computeCamelDependencies() string { } results := make([]string, 0, len(catalog.Artifacts)) - for k := range catalog.Artifacts { - results = append(results, k) + for a := range catalog.Artifacts { + // skipping camel-k-* and other artifacts as they may not be useful for cli completion + if strings.HasPrefix(a, "camel-quarkus-") { + results = append(results, v1.NormalizeDependency(a)) + } } + sort.Strings(results) return strings.Join(results, " ") } From 783d8046797efbe4455469511f5d68c701abbd2a Mon Sep 17 00:00:00 2001 From: Tadayoshi Sato Date: Sat, 20 Aug 2022 14:51:12 +0900 Subject: [PATCH 03/11] feat(cli): add completion support for kamel local -d flag --- pkg/cmd/local.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/cmd/local.go b/pkg/cmd/local.go index c5616d5ed9..fd1b2d05a2 100644 --- a/pkg/cmd/local.go +++ b/pkg/cmd/local.go @@ -58,6 +58,9 @@ func newCmdLocal(rootCmdOptions *RootCmdOptions) (*cobra.Command, *LocalCmdOptio fmt.Fprintln(cmd.ErrOrStderr(), err.Error()) } + // completion support + configureKnownCompletions(&cmd) + cmd.AddCommand(cmdOnly(newCmdLocalBuild(&options))) cmd.AddCommand(cmdOnly(newCmdLocalInspect(&options))) cmd.AddCommand(cmdOnly(newCmdLocalRun(&options))) From 6059e2e178040484091bc9bbec1b9815c3deceb6 Mon Sep 17 00:00:00 2001 From: Tadayoshi Sato Date: Thu, 25 Aug 2022 15:08:41 +0900 Subject: [PATCH 04/11] feat(cli): kamel local - show warn message for missing Camel dependency --- e2e/local/local_build_test.go | 33 ++++++ e2e/local/local_inspect_test.go | 101 ++++++++++++++++++ e2e/local/local_run_test.go | 44 ++++++-- .../camel/v1/camelcatalog_types_support.go | 10 ++ .../camel/v1/integration_types_support.go | 4 + pkg/builder/quarkus.go | 5 +- pkg/cmd/local.go | 2 +- pkg/cmd/local/local.go | 41 +++++-- pkg/cmd/local_build.go | 16 ++- pkg/cmd/local_inspect.go | 17 ++- pkg/cmd/local_run.go | 20 ++-- pkg/util/camel/camel_runtime_catalog.go | 13 ++- 12 files changed, 254 insertions(+), 52 deletions(-) create mode 100644 e2e/local/local_inspect_test.go diff --git a/e2e/local/local_build_test.go b/e2e/local/local_build_test.go index 7b0ea7f51e..54d9be9a6d 100644 --- a/e2e/local/local_build_test.go +++ b/e2e/local/local_build_test.go @@ -25,6 +25,7 @@ import ( "fmt" "io" "strings" + "sync" "testing" . "github.com/onsi/gomega" @@ -113,6 +114,38 @@ func TestLocalBuildWithTrait(t *testing.T) { Eventually(DockerImages, TestTimeoutMedium).Should(ContainSubstring(image)) } +func TestLocalBuildWithInvalidDependency(t *testing.T) { + RegisterTestingT(t) + + ctx, cancel := context.WithCancel(TestContext) + defer cancel() + piper, pipew := io.Pipe() + defer pipew.Close() + defer piper.Close() + + file := testutil.MakeTempCopy(t, "files/yaml.yaml") + image := "test/test-" + strings.ToLower(util.RandomString(10)) + + kamelBuild := KamelWithContext(ctx, "local", "build", file, "--image", image, "-d", "camel-xxx") + kamelBuild.SetOut(pipew) + kamelBuild.SetErr(pipew) + + logScanner := testutil.NewLogScanner(ctx, piper, "Warning: dependency camel:xxx not found in Camel catalog") + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + err := kamelBuild.Execute() + assert.Error(t, err) + cancel() + }() + + Eventually(logScanner.IsFound("Warning: dependency camel:xxx not found in Camel catalog"), TestTimeoutShort). + Should(BeTrue()) + wg.Wait() +} + func TestLocalBuildIntegrationDirectory(t *testing.T) { RegisterTestingT(t) diff --git a/e2e/local/local_inspect_test.go b/e2e/local/local_inspect_test.go new file mode 100644 index 0000000000..92dbec905f --- /dev/null +++ b/e2e/local/local_inspect_test.go @@ -0,0 +1,101 @@ +//go:build integration +// +build integration + +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package local + +import ( + "context" + "io" + "testing" + + . "github.com/onsi/gomega" + "github.com/stretchr/testify/assert" + + . "github.com/apache/camel-k/e2e/support" + testutil "github.com/apache/camel-k/e2e/support/util" +) + +func TestLocalInspect(t *testing.T) { + RegisterTestingT(t) + + ctx, cancel := context.WithCancel(TestContext) + defer cancel() + piper, pipew := io.Pipe() + defer pipew.Close() + defer piper.Close() + + file := testutil.MakeTempCopy(t, "files/yaml.yaml") + + kamelInspect := KamelWithContext(ctx, "local", "inspect", file) + kamelInspect.SetOut(pipew) + kamelInspect.SetErr(pipew) + + logScanner := testutil.NewLogScanner(ctx, piper, + "camel:log", + "camel:timer", + "mvn:org.apache.camel.quarkus:camel-quarkus-yaml-dsl", + ) + + go func() { + err := kamelInspect.Execute() + assert.NoError(t, err) + cancel() + }() + + Eventually(logScanner.IsFound("camel:log"), TestTimeoutShort).Should(BeTrue()) + Eventually(logScanner.IsFound("camel:timer"), TestTimeoutShort).Should(BeTrue()) + Eventually(logScanner.IsFound("mvn:org.apache.camel.quarkus:camel-quarkus-yaml-dsl"), TestTimeoutShort). + Should(BeTrue()) +} + +func TestLocalInspectWithDependencies(t *testing.T) { + RegisterTestingT(t) + + ctx, cancel := context.WithCancel(TestContext) + defer cancel() + piper, pipew := io.Pipe() + defer pipew.Close() + defer piper.Close() + + file := testutil.MakeTempCopy(t, "files/yaml.yaml") + + kamelInspect := KamelWithContext(ctx, "local", "inspect", file, "-d", "camel-amqp", "-d", "camel-xxx") + kamelInspect.SetOut(pipew) + kamelInspect.SetErr(pipew) + + logScanner := testutil.NewLogScanner(ctx, piper, + "Warning: dependency camel:xxx not found in Camel catalog", + "camel:amqp", + "camel:log", + "camel:timer", + ) + + go func() { + err := kamelInspect.Execute() + assert.NoError(t, err) + cancel() + }() + + Eventually(logScanner.IsFound("Warning: dependency camel:xxx not found in Camel catalog"), TestTimeoutShort). + Should(BeTrue()) + Eventually(logScanner.IsFound("camel:amqp"), TestTimeoutShort).Should(BeTrue()) + Eventually(logScanner.IsFound("camel:log"), TestTimeoutShort).Should(BeTrue()) + Eventually(logScanner.IsFound("camel:timer"), TestTimeoutShort).Should(BeTrue()) +} diff --git a/e2e/local/local_run_test.go b/e2e/local/local_run_test.go index 44a9363958..3c3dbd49e6 100644 --- a/e2e/local/local_run_test.go +++ b/e2e/local/local_run_test.go @@ -24,6 +24,7 @@ import ( "context" "io" "strings" + "sync" "testing" . "github.com/onsi/gomega" @@ -52,8 +53,7 @@ func TestLocalRun(t *testing.T) { logScanner := testutil.NewLogScanner(ctx, piper, "Magicstring!") go func() { - err := kamelRun.Execute() - assert.NoError(t, err) + _ = kamelRun.Execute() cancel() }() @@ -78,14 +78,44 @@ func TestLocalRunWithDependencies(t *testing.T) { logScanner := testutil.NewLogScanner(ctx, piper, "Magicstring!") go func() { - err := kamelRun.Execute() - assert.NoError(t, err) + _ = kamelRun.Execute() cancel() }() Eventually(logScanner.IsFound("Magicstring!"), TestTimeoutMedium).Should(BeTrue()) } +func TestLocalRunWithInvalidDependency(t *testing.T) { + RegisterTestingT(t) + + ctx, cancel := context.WithCancel(TestContext) + defer cancel() + piper, pipew := io.Pipe() + defer pipew.Close() + defer piper.Close() + + file := testutil.MakeTempCopy(t, "files/yaml.yaml") + + kamelRun := KamelWithContext(ctx, "local", "run", file, "-d", "camel-xxx") + kamelRun.SetOut(pipew) + kamelRun.SetErr(pipew) + + logScanner := testutil.NewLogScanner(ctx, piper, "Warning: dependency camel:xxx not found in Camel catalog") + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + err := kamelRun.Execute() + assert.Error(t, err) + cancel() + }() + + Eventually(logScanner.IsFound("Warning: dependency camel:xxx not found in Camel catalog"), TestTimeoutShort). + Should(BeTrue()) + wg.Wait() +} + func TestLocalRunContainerize(t *testing.T) { RegisterTestingT(t) @@ -106,8 +136,7 @@ func TestLocalRunContainerize(t *testing.T) { defer StopDockerContainers() go func() { - err := kamelRun.Execute() - assert.NoError(t, err) + _ = kamelRun.Execute() cancel() }() @@ -149,8 +178,7 @@ func TestLocalRunIntegrationDirectory(t *testing.T) { logScanner := testutil.NewLogScanner(ctx2, piper, "Magicstring!") go func() { - err := kamelRun.Execute() - assert.NoError(t, err) + _ = kamelRun.Execute() cancel2() }() diff --git a/pkg/apis/camel/v1/camelcatalog_types_support.go b/pkg/apis/camel/v1/camelcatalog_types_support.go index 777d6bc804..1106a8b51d 100644 --- a/pkg/apis/camel/v1/camelcatalog_types_support.go +++ b/pkg/apis/camel/v1/camelcatalog_types_support.go @@ -72,6 +72,16 @@ func (c *CamelCatalogSpec) GetCamelVersion() string { return c.Runtime.Metadata["camel.version"] } +// GetCamelQuarkusVersion returns the Camel Quarkus version the runtime is based on. +func (c *CamelCatalogSpec) GetCamelQuarkusVersion() string { + return c.Runtime.Metadata["camel-quarkus.version"] +} + +// GetQuarkusVersion returns the Quarkus version the runtime is based on. +func (c *CamelCatalogSpec) GetQuarkusVersion() string { + return c.Runtime.Metadata["quarkus.version"] +} + // HasCapability checks if the given capability is present in the catalog. func (c *CamelCatalogSpec) HasCapability(capability string) bool { _, ok := c.Runtime.Capabilities[capability] diff --git a/pkg/apis/camel/v1/integration_types_support.go b/pkg/apis/camel/v1/integration_types_support.go index 3885b92118..7a0aff91e8 100644 --- a/pkg/apis/camel/v1/integration_types_support.go +++ b/pkg/apis/camel/v1/integration_types_support.go @@ -124,6 +124,10 @@ func NormalizeDependency(dependency string) string { newDep = "camel:" + strings.TrimPrefix(dependency, "camel-quarkus-") case strings.HasPrefix(newDep, "camel-quarkus:"): newDep = "camel:" + strings.TrimPrefix(dependency, "camel-quarkus:") + case strings.HasPrefix(newDep, "camel-k-"): + newDep = "camel-k:" + strings.TrimPrefix(dependency, "camel-k-") + case strings.HasPrefix(newDep, "camel-k:"): + // do nothing case strings.HasPrefix(newDep, "camel-"): newDep = "camel:" + strings.TrimPrefix(dependency, "camel-") } diff --git a/pkg/builder/quarkus.go b/pkg/builder/quarkus.go index 34331c6ca0..2061a160fc 100644 --- a/pkg/builder/quarkus.go +++ b/pkg/builder/quarkus.go @@ -78,7 +78,10 @@ func loadCamelQuarkusCatalog(ctx *builderContext) error { } func generateQuarkusProject(ctx *builderContext) error { - p := GenerateQuarkusProjectCommon(ctx.Build.Runtime.Metadata["camel-quarkus.version"], ctx.Build.Runtime.Version, ctx.Build.Runtime.Metadata["quarkus.version"]) + p := GenerateQuarkusProjectCommon( + ctx.Build.Runtime.Metadata["camel-quarkus.version"], + ctx.Build.Runtime.Version, + ctx.Build.Runtime.Metadata["quarkus.version"]) // Add all the properties from the build configuration p.Properties.AddAll(ctx.Build.Maven.Properties) diff --git a/pkg/cmd/local.go b/pkg/cmd/local.go index fd1b2d05a2..eece67043f 100644 --- a/pkg/cmd/local.go +++ b/pkg/cmd/local.go @@ -89,7 +89,7 @@ func (o *LocalCmdOptions) persistentPreRun(cmd *cobra.Command, args []string) er func warnTraitUsages(cmd *cobra.Command, traits []string) { if len(traits) > 0 { - fmt.Fprintf(cmd.OutOrStdout(), + fmt.Fprintf(cmd.ErrOrStderr(), "Warning: traits are specified but don't take effect for local run: %v\n", traits) } } diff --git a/pkg/cmd/local/local.go b/pkg/cmd/local/local.go index 9ba54dd660..56dae1f6b0 100644 --- a/pkg/cmd/local/local.go +++ b/pkg/cmd/local/local.go @@ -24,6 +24,7 @@ import ( "io/ioutil" "os" "path" + "sort" "strings" "github.com/pkg/errors" @@ -47,21 +48,25 @@ var acceptedDependencyTypes = []string{ } // GetDependencies resolves and gets the list of dependencies from catalog and sources. -func GetDependencies(ctx context.Context, args []string, additionalDependencies []string, repositories []string, allDependencies bool) ([]string, error) { +func GetDependencies(ctx context.Context, cmd *cobra.Command, srcs, userDependencies, repositories []string, + allDependencies bool) ([]string, error) { // Fetch existing catalog or create new one if one does not already exist catalog, err := createCamelCatalog(ctx) if err != nil { return nil, errors.Wrap(err, "failed to create Camel catalog") } - // Get top-level dependencies - dependencies, err := getTopLevelDependencies(ctx, catalog, args) + // Validate user-provided dependencies against Camel catalog + validateCamelDependency(cmd, catalog, userDependencies) + + // Get top-level dependencies from sources + dependencies, err := getTopLevelDependencies(ctx, catalog, srcs) if err != nil { - return nil, errors.Wrap(err, "failed to get top-level dependencies") + return nil, errors.Wrap(err, "failed to resolve dependencies from source") } // Add additional user-provided dependencies - dependencies = append(dependencies, additionalDependencies...) + dependencies = append(dependencies, userDependencies...) // Compute transitive dependencies if allDependencies { @@ -80,12 +85,13 @@ func GetDependencies(ctx context.Context, args []string, additionalDependencies return dependencies, nil } -func getTopLevelDependencies(ctx context.Context, catalog *camel.RuntimeCatalog, args []string) ([]string, error) { +// getTopLevelDependencies gets top-level dependencies from sources. +func getTopLevelDependencies(ctx context.Context, catalog *camel.RuntimeCatalog, srcs []string) ([]string, error) { // List of top-level dependencies dependencies := strset.New() // Invoke the dependency inspector code for each source file - for _, src := range args { + for _, src := range srcs { data, _, _, err := source.LoadTextContent(ctx, src, false) if err != nil { return []string{}, err @@ -110,11 +116,27 @@ func getTopLevelDependencies(ctx context.Context, catalog *camel.RuntimeCatalog, return dependencies.List(), nil } +// validateCamelDependency validates dependencies against Camel catalog. +// It only shows warning and does not throw error in case the Catalog is just not complete +// and we don't want to let it stop the process. +func validateCamelDependency(cmd *cobra.Command, catalog *camel.RuntimeCatalog, dependencies []string) { + for _, d := range dependencies { + if !strings.HasPrefix(d, "camel:") { + continue + } + + artifact := strings.TrimPrefix(d, "camel:") + if ok := catalog.HasArtifact(artifact); !ok { + fmt.Fprintf(cmd.ErrOrStderr(), "Warning: dependency %s not found in Camel catalog\n", d) + } + } +} + func getTransitiveDependencies(ctx context.Context, catalog *camel.RuntimeCatalog, dependencies []string, repositories []string) ([]string, error) { project := builder.GenerateQuarkusProjectCommon( - catalog.CamelCatalogSpec.Runtime.Metadata["camel-quarkus.version"], + catalog.GetCamelQuarkusVersion(), defaults.DefaultRuntimeVersion, - catalog.CamelCatalogSpec.Runtime.Metadata["quarkus.version"], + catalog.GetQuarkusVersion(), ) if err := camel.ManageIntegrationDependencies(&project, dependencies, catalog); err != nil { @@ -242,6 +264,7 @@ func createCamelCatalog(ctx context.Context) (*camel.RuntimeCatalog, error) { } func OutputDependencies(dependencies []string, format string, cmd *cobra.Command) error { + sort.Strings(dependencies) if format != "" { if err := printDependencies(format, dependencies, cmd); err != nil { return err diff --git a/pkg/cmd/local_build.go b/pkg/cmd/local_build.go index 6f8cbcfd03..6bd124a1e8 100644 --- a/pkg/cmd/local_build.go +++ b/pkg/cmd/local_build.go @@ -44,14 +44,12 @@ func newCmdLocalBuild(localCmdOptions *LocalCmdOptions) (*cobra.Command, *localB if err := options.init(args); err != nil { return err } - if err := options.run(cmd, args); err != nil { - fmt.Fprintln(cmd.ErrOrStderr(), err.Error()) - } - if err := options.deinit(); err != nil { - return err - } - - return nil + defer func() { + if err := options.deinit(); err != nil { + fmt.Fprintln(cmd.ErrOrStderr(), err.Error()) + } + }() + return options.run(cmd, args) }, Annotations: map[string]string{ offlineCommandLabel: "true", @@ -187,7 +185,7 @@ func (o *localBuildCmdOptions) run(cmd *cobra.Command, args []string) error { routeFiles := args if !o.BaseImage { - dependencies, err := local.GetDependencies(o.Context, args, o.Dependencies, o.MavenRepositories, true) + dependencies, err := local.GetDependencies(o.Context, cmd, args, o.Dependencies, o.MavenRepositories, true) if err != nil { return err } diff --git a/pkg/cmd/local_inspect.go b/pkg/cmd/local_inspect.go index 553f36fcaa..7b47e3374e 100644 --- a/pkg/cmd/local_inspect.go +++ b/pkg/cmd/local_inspect.go @@ -44,14 +44,12 @@ will be generated by calling Maven and then printed in the selected output forma if err := options.init(); err != nil { return err } - if err := options.run(cmd, args); err != nil { - fmt.Fprintln(cmd.ErrOrStderr(), err.Error()) - } - if err := options.deinit(); err != nil { - return err - } - - return nil + defer func() { + if err := options.deinit(); err != nil { + fmt.Fprintln(cmd.ErrOrStderr(), err.Error()) + } + }() + return options.run(cmd, args) }, Annotations: map[string]string{ offlineCommandLabel: "true", @@ -92,7 +90,8 @@ func (o *localInspectCmdOptions) init() error { } func (o *localInspectCmdOptions) run(cmd *cobra.Command, args []string) error { - dependencies, err := local.GetDependencies(o.Context, args, o.Dependencies, o.MavenRepositories, o.AllDependencies) + dependencies, err := local.GetDependencies(o.Context, cmd, + args, o.Dependencies, o.MavenRepositories, o.AllDependencies) if err != nil { return err } diff --git a/pkg/cmd/local_run.go b/pkg/cmd/local_run.go index 2c3ce54a7d..e88ff0903b 100644 --- a/pkg/cmd/local_run.go +++ b/pkg/cmd/local_run.go @@ -59,14 +59,12 @@ func newCmdLocalRun(localCmdOptions *LocalCmdOptions) (*cobra.Command, *localRun os.Exit(0) }() - if err := options.run(cmd, args); err != nil { - fmt.Fprintln(cmd.ErrOrStderr(), err.Error()) - } - if err := options.deinit(); err != nil { - return err - } - - return nil + defer func() { + if err := options.deinit(); err != nil { + fmt.Fprintln(cmd.ErrOrStderr(), err.Error()) + } + }() + return options.run(cmd, args) }, Annotations: map[string]string{ offlineCommandLabel: "true", @@ -153,7 +151,7 @@ func (o *localRunCmdOptions) run(cmd *cobra.Command, args []string) error { return local.RunIntegrationImage(o.Context, o.Image, cmd.OutOrStdout(), cmd.ErrOrStderr()) } - dependencies, err := o.processDependencies(args) + dependencies, err := o.processDependencies(cmd, args) if err != nil { return err } @@ -182,9 +180,9 @@ func (o *localRunCmdOptions) run(cmd *cobra.Command, args []string) error { cmd.OutOrStdout(), cmd.ErrOrStderr()) } -func (o *localRunCmdOptions) processDependencies(args []string) ([]string, error) { +func (o *localRunCmdOptions) processDependencies(cmd *cobra.Command, args []string) ([]string, error) { if o.IntegrationDirectory == "" { - return local.GetDependencies(o.Context, args, o.Dependencies, o.MavenRepositories, true) + return local.GetDependencies(o.Context, cmd, args, o.Dependencies, o.MavenRepositories, true) } // Set up on the integration directory diff --git a/pkg/util/camel/camel_runtime_catalog.go b/pkg/util/camel/camel_runtime_catalog.go index 787370a5fd..f3a6b37592 100644 --- a/pkg/util/camel/camel_runtime_catalog.go +++ b/pkg/util/camel/camel_runtime_catalog.go @@ -80,13 +80,18 @@ type RuntimeCatalog struct { javaTypeDependencies map[string]string } -// HasArtifact --. +// HasArtifact checks if the given artifact is present in the catalog. func (c *RuntimeCatalog) HasArtifact(artifact string) bool { - if !strings.HasPrefix(artifact, "camel-") { - artifact = "camel-" + artifact + a := artifact + if !strings.HasPrefix(a, "camel-") { + if c.Runtime.Provider == v1.RuntimeProviderQuarkus { + a = "camel-quarkus-" + a + } else { + a = "camel-" + a + } } - _, ok := c.Artifacts[artifact] + _, ok := c.Artifacts[a] return ok } From 3883e885e6355a4596e497cae44ef94bf959e4d4 Mon Sep 17 00:00:00 2001 From: Tadayoshi Sato Date: Thu, 25 Aug 2022 15:14:41 +0900 Subject: [PATCH 05/11] feat(cli): kamel run - show warn message for missing Camel dependency --- .../camel/v1/integration_types_support.go | 25 +-------- .../v1/integration_types_support_test.go | 12 ++--- pkg/cmd/completion_bash.go | 2 +- pkg/cmd/local.go | 4 +- pkg/cmd/local/local.go | 22 ++------ pkg/cmd/run.go | 16 +++++- pkg/cmd/{run_help.go => run_support.go} | 7 +++ .../{run_help_test.go => run_support_test.go} | 46 +++++++++++++++++ pkg/cmd/run_test.go | 45 ---------------- pkg/util/camel/camel_dependencies.go | 51 ++++++++++++++++++- pkg/util/camel/camel_dependencies_test.go | 33 ++++++++++++ 11 files changed, 163 insertions(+), 100 deletions(-) rename pkg/cmd/{run_help.go => run_support.go} (96%) rename pkg/cmd/{run_help_test.go => run_support_test.go} (56%) create mode 100644 pkg/util/camel/camel_dependencies_test.go diff --git a/pkg/apis/camel/v1/integration_types_support.go b/pkg/apis/camel/v1/integration_types_support.go index 7a0aff91e8..8eb70ef8bb 100644 --- a/pkg/apis/camel/v1/integration_types_support.go +++ b/pkg/apis/camel/v1/integration_types_support.go @@ -105,33 +105,12 @@ func (in *IntegrationSpec) AddDependency(dependency string) { if in.Dependencies == nil { in.Dependencies = make([]string, 0) } - newDep := NormalizeDependency(dependency) for _, d := range in.Dependencies { - if d == newDep { + if d == dependency { return } } - in.Dependencies = append(in.Dependencies, newDep) -} - -// NormalizeDependency converts different forms of camel dependencies -// -- `camel-xxx`, `camel-quarkus-xxx`, and `camel-quarkus:xxx` -- -// into the unified form `camel:xxx`. -func NormalizeDependency(dependency string) string { - newDep := dependency - switch { - case strings.HasPrefix(newDep, "camel-quarkus-"): - newDep = "camel:" + strings.TrimPrefix(dependency, "camel-quarkus-") - case strings.HasPrefix(newDep, "camel-quarkus:"): - newDep = "camel:" + strings.TrimPrefix(dependency, "camel-quarkus:") - case strings.HasPrefix(newDep, "camel-k-"): - newDep = "camel-k:" + strings.TrimPrefix(dependency, "camel-k-") - case strings.HasPrefix(newDep, "camel-k:"): - // do nothing - case strings.HasPrefix(newDep, "camel-"): - newDep = "camel:" + strings.TrimPrefix(dependency, "camel-") - } - return newDep + in.Dependencies = append(in.Dependencies, dependency) } // GetConfigurationProperty returns a configuration property diff --git a/pkg/apis/camel/v1/integration_types_support_test.go b/pkg/apis/camel/v1/integration_types_support_test.go index 7a5e81d860..81b1c59734 100644 --- a/pkg/apis/camel/v1/integration_types_support_test.go +++ b/pkg/apis/camel/v1/integration_types_support_test.go @@ -62,7 +62,10 @@ func TestLanguageAlreadySet(t *testing.T) { func TestAddDependency(t *testing.T) { integration := IntegrationSpec{} - integration.AddDependency("camel-file") + integration.AddDependency("camel:file") + assert.Equal(t, integration.Dependencies, []string{"camel:file"}) + // adding the same dependency twice won't duplicate it in the list + integration.AddDependency("camel:file") assert.Equal(t, integration.Dependencies, []string{"camel:file"}) integration = IntegrationSpec{} @@ -74,13 +77,6 @@ func TestAddDependency(t *testing.T) { assert.Equal(t, integration.Dependencies, []string{"file:dep"}) } -func TestNormalizeDependency(t *testing.T) { - assert.Equal(t, "camel:file", NormalizeDependency("camel-file")) - assert.Equal(t, "camel:file", NormalizeDependency("camel:file")) - assert.Equal(t, "camel:file", NormalizeDependency("camel-quarkus-file")) - assert.Equal(t, "camel:file", NormalizeDependency("camel-quarkus:file")) -} - func TestGetConfigurationProperty(t *testing.T) { integration := IntegrationSpec{} integration.AddConfiguration("property", "key1=value1") diff --git a/pkg/cmd/completion_bash.go b/pkg/cmd/completion_bash.go index 1f85aaf8bf..2b04ccec2b 100644 --- a/pkg/cmd/completion_bash.go +++ b/pkg/cmd/completion_bash.go @@ -301,7 +301,7 @@ func computeCamelDependencies() string { for a := range catalog.Artifacts { // skipping camel-k-* and other artifacts as they may not be useful for cli completion if strings.HasPrefix(a, "camel-quarkus-") { - results = append(results, v1.NormalizeDependency(a)) + results = append(results, camel.NormalizeDependency(a)) } } sort.Strings(results) diff --git a/pkg/cmd/local.go b/pkg/cmd/local.go index eece67043f..70a0f7a15d 100644 --- a/pkg/cmd/local.go +++ b/pkg/cmd/local.go @@ -20,7 +20,7 @@ package cmd import ( "fmt" - v1 "github.com/apache/camel-k/pkg/apis/camel/v1" + "github.com/apache/camel-k/pkg/util/camel" "github.com/spf13/cobra" ) @@ -78,7 +78,7 @@ type LocalCmdOptions struct { func (o *LocalCmdOptions) persistentPreRun(cmd *cobra.Command, args []string) error { // pre-process dependencies for i, dependency := range o.Dependencies { - o.Dependencies[i] = v1.NormalizeDependency(dependency) + o.Dependencies[i] = camel.NormalizeDependency(dependency) } // validate traits diff --git a/pkg/cmd/local/local.go b/pkg/cmd/local/local.go index 56dae1f6b0..353e7a1f97 100644 --- a/pkg/cmd/local/local.go +++ b/pkg/cmd/local/local.go @@ -51,13 +51,13 @@ var acceptedDependencyTypes = []string{ func GetDependencies(ctx context.Context, cmd *cobra.Command, srcs, userDependencies, repositories []string, allDependencies bool) ([]string, error) { // Fetch existing catalog or create new one if one does not already exist - catalog, err := createCamelCatalog(ctx) + catalog, err := CreateCamelCatalog(ctx) if err != nil { return nil, errors.Wrap(err, "failed to create Camel catalog") } // Validate user-provided dependencies against Camel catalog - validateCamelDependency(cmd, catalog, userDependencies) + camel.ValidateDependencies(catalog, userDependencies, cmd) // Get top-level dependencies from sources dependencies, err := getTopLevelDependencies(ctx, catalog, srcs) @@ -116,22 +116,6 @@ func getTopLevelDependencies(ctx context.Context, catalog *camel.RuntimeCatalog, return dependencies.List(), nil } -// validateCamelDependency validates dependencies against Camel catalog. -// It only shows warning and does not throw error in case the Catalog is just not complete -// and we don't want to let it stop the process. -func validateCamelDependency(cmd *cobra.Command, catalog *camel.RuntimeCatalog, dependencies []string) { - for _, d := range dependencies { - if !strings.HasPrefix(d, "camel:") { - continue - } - - artifact := strings.TrimPrefix(d, "camel:") - if ok := catalog.HasArtifact(artifact); !ok { - fmt.Fprintf(cmd.ErrOrStderr(), "Warning: dependency %s not found in Camel catalog\n", d) - } - } -} - func getTransitiveDependencies(ctx context.Context, catalog *camel.RuntimeCatalog, dependencies []string, repositories []string) ([]string, error) { project := builder.GenerateQuarkusProjectCommon( catalog.GetCamelQuarkusVersion(), @@ -245,7 +229,7 @@ func generateCatalog(ctx context.Context) (*camel.RuntimeCatalog, error) { return catalog, nil } -func createCamelCatalog(ctx context.Context) (*camel.RuntimeCatalog, error) { +func CreateCamelCatalog(ctx context.Context) (*camel.RuntimeCatalog, error) { // Attempt to reuse existing Camel catalog if one is present catalog, err := camel.DefaultCatalog() if err != nil { diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index 84c4d6f342..72460acd92 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -66,6 +66,7 @@ import ( "github.com/apache/camel-k/pkg/platform" "github.com/apache/camel-k/pkg/trait" "github.com/apache/camel-k/pkg/util" + "github.com/apache/camel-k/pkg/util/camel" "github.com/apache/camel-k/pkg/util/defaults" "github.com/apache/camel-k/pkg/util/dsl" "github.com/apache/camel-k/pkg/util/kubernetes" @@ -797,6 +798,7 @@ func (o *runCmdOptions) convertToTraitParameter(c client.Client, value, traitPar func (o *runCmdOptions) applyDependencies(cmd *cobra.Command, c client.Client, it *v1.Integration, name string) error { var platform *v1.IntegrationPlatform + var catalog *camel.RuntimeCatalog for _, item := range o.Dependencies { if strings.HasPrefix(item, "file://") || strings.HasPrefix(item, "http://") || strings.HasPrefix(item, "https://") { if platform == nil { @@ -810,7 +812,19 @@ func (o *runCmdOptions) applyDependencies(cmd *cobra.Command, c client.Client, i return errors.Wrap(err, fmt.Sprintf("Error trying to upload %s to the Image Registry.", item)) } } else { - it.Spec.AddDependency(item) + if catalog == nil { + // The catalog used for lightweight validation of Camel components. + // The exact runtime version is not used here since resolving the runtime version may be + // a costly operation and most of the use cases should be covered by the default catalog. + // And the validation only warns potential misusages of Camel components at the CLI level, + // so strictness of catalog version is not necessary here. + var err error + catalog, err = local.CreateCamelCatalog(o.Context) + if err != nil { + return err + } + } + addDependency(cmd, it, item, catalog) } } diff --git a/pkg/cmd/run_help.go b/pkg/cmd/run_support.go similarity index 96% rename from pkg/cmd/run_help.go rename to pkg/cmd/run_support.go index ce0aa060db..a25ca31b3f 100644 --- a/pkg/cmd/run_help.go +++ b/pkg/cmd/run_support.go @@ -32,6 +32,7 @@ import ( v1 "github.com/apache/camel-k/pkg/apis/camel/v1" "github.com/apache/camel-k/pkg/client" "github.com/apache/camel-k/pkg/cmd/source" + "github.com/apache/camel-k/pkg/util/camel" "github.com/apache/camel-k/pkg/util/kubernetes" "github.com/apache/camel-k/pkg/util/resource" "github.com/magiconair/properties" @@ -39,6 +40,12 @@ import ( corev1 "k8s.io/api/core/v1" ) +func addDependency(cmd *cobra.Command, it *v1.Integration, dependency string, catalog *camel.RuntimeCatalog) { + normalized := camel.NormalizeDependency(dependency) + camel.ValidateDependency(catalog, normalized, cmd) + it.Spec.AddDependency(normalized) +} + func parseConfigAndGenCm(ctx context.Context, cmd *cobra.Command, c client.Client, config *resource.Config, integration *v1.Integration, enableCompression bool) (*corev1.ConfigMap, error) { switch config.StorageType() { case resource.StorageTypeConfigmap: diff --git a/pkg/cmd/run_help_test.go b/pkg/cmd/run_support_test.go similarity index 56% rename from pkg/cmd/run_help_test.go rename to pkg/cmd/run_support_test.go index 15ff0d93f1..e0a536c7af 100644 --- a/pkg/cmd/run_help_test.go +++ b/pkg/cmd/run_support_test.go @@ -24,9 +24,55 @@ import ( "strings" "testing" + v1 "github.com/apache/camel-k/pkg/apis/camel/v1" + "github.com/apache/camel-k/pkg/cmd/source" "github.com/stretchr/testify/assert" ) +func TestRunBinaryResource(t *testing.T) { + binaryResourceSpec, err := binaryOrTextResource("file.ext", []byte{1, 2, 3, 4}, "application/octet-stream", false, v1.ResourceTypeData, "") + assert.Nil(t, err) + assert.Equal(t, "", binaryResourceSpec.Content) + assert.NotNil(t, binaryResourceSpec.RawContent) + assert.Equal(t, "file.ext", binaryResourceSpec.Name) + assert.Equal(t, "application/octet-stream", binaryResourceSpec.ContentType) + assert.False(t, binaryResourceSpec.Compression) +} + +func TestRunBinaryCompressedResource(t *testing.T) { + data := []byte{1, 2, 3, 4} + base64Compressed, _ := source.CompressToString(data) + binaryResourceSpec, err := binaryOrTextResource("file.ext", data, "application/octet-stream", true, v1.ResourceTypeData, "") + assert.Nil(t, err) + assert.Equal(t, base64Compressed, binaryResourceSpec.Content) + assert.Nil(t, binaryResourceSpec.RawContent) + assert.Equal(t, "file.ext", binaryResourceSpec.Name) + assert.Equal(t, "application/octet-stream", binaryResourceSpec.ContentType) + assert.True(t, binaryResourceSpec.Compression) +} + +func TestRunTextResource(t *testing.T) { + textResourceSpec, err := binaryOrTextResource("file.ext", []byte("hello world"), "text/plain", false, v1.ResourceTypeData, "") + assert.Nil(t, err) + assert.Equal(t, "hello world", textResourceSpec.Content) + assert.Nil(t, textResourceSpec.RawContent) + assert.Equal(t, "file.ext", textResourceSpec.Name) + assert.Equal(t, "text/plain", textResourceSpec.ContentType) + assert.False(t, textResourceSpec.Compression) +} + +func TestRunTextCompressedResource(t *testing.T) { + data := []byte("hello horld") + base64Compressed, _ := source.CompressToString(data) + textResourceSpec, err := binaryOrTextResource("file.ext", []byte("hello horld"), "text/plain", true, v1.ResourceTypeData, "") + assert.Nil(t, err) + assert.Equal(t, base64Compressed, textResourceSpec.Content) + assert.Nil(t, textResourceSpec.RawContent) + assert.Equal(t, "file.ext", textResourceSpec.Name) + assert.Equal(t, "text/plain", textResourceSpec.ContentType) + assert.True(t, textResourceSpec.Compression) +} + func TestFilterFileLocation(t *testing.T) { optionFileLocations := []string{ "file:/path/to/valid/file", diff --git a/pkg/cmd/run_test.go b/pkg/cmd/run_test.go index 2b7144071b..8d2969df7a 100644 --- a/pkg/cmd/run_test.go +++ b/pkg/cmd/run_test.go @@ -27,7 +27,6 @@ import ( v1 "github.com/apache/camel-k/pkg/apis/camel/v1" traitv1 "github.com/apache/camel-k/pkg/apis/camel/v1/trait" - "github.com/apache/camel-k/pkg/cmd/source" "github.com/apache/camel-k/pkg/platform" "github.com/apache/camel-k/pkg/trait" "github.com/apache/camel-k/pkg/util/test" @@ -517,50 +516,6 @@ func TestRunValidateArgs(t *testing.T) { assert.Equal(t, "One of the provided sources is not reachable: missing file or unsupported scheme in missing_file", err.Error()) } -func TestRunBinaryResource(t *testing.T) { - binaryResourceSpec, err := binaryOrTextResource("file.ext", []byte{1, 2, 3, 4}, "application/octet-stream", false, v1.ResourceTypeData, "") - assert.Nil(t, err) - assert.Equal(t, "", binaryResourceSpec.Content) - assert.NotNil(t, binaryResourceSpec.RawContent) - assert.Equal(t, "file.ext", binaryResourceSpec.Name) - assert.Equal(t, "application/octet-stream", binaryResourceSpec.ContentType) - assert.False(t, binaryResourceSpec.Compression) -} - -func TestRunBinaryCompressedResource(t *testing.T) { - data := []byte{1, 2, 3, 4} - base64Compressed, _ := source.CompressToString(data) - binaryResourceSpec, err := binaryOrTextResource("file.ext", data, "application/octet-stream", true, v1.ResourceTypeData, "") - assert.Nil(t, err) - assert.Equal(t, base64Compressed, binaryResourceSpec.Content) - assert.Nil(t, binaryResourceSpec.RawContent) - assert.Equal(t, "file.ext", binaryResourceSpec.Name) - assert.Equal(t, "application/octet-stream", binaryResourceSpec.ContentType) - assert.True(t, binaryResourceSpec.Compression) -} - -func TestRunTextResource(t *testing.T) { - textResourceSpec, err := binaryOrTextResource("file.ext", []byte("hello world"), "text/plain", false, v1.ResourceTypeData, "") - assert.Nil(t, err) - assert.Equal(t, "hello world", textResourceSpec.Content) - assert.Nil(t, textResourceSpec.RawContent) - assert.Equal(t, "file.ext", textResourceSpec.Name) - assert.Equal(t, "text/plain", textResourceSpec.ContentType) - assert.False(t, textResourceSpec.Compression) -} - -func TestRunTextCompressedResource(t *testing.T) { - data := []byte("hello horld") - base64Compressed, _ := source.CompressToString(data) - textResourceSpec, err := binaryOrTextResource("file.ext", []byte("hello horld"), "text/plain", true, v1.ResourceTypeData, "") - assert.Nil(t, err) - assert.Equal(t, base64Compressed, textResourceSpec.Content) - assert.Nil(t, textResourceSpec.RawContent) - assert.Equal(t, "file.ext", textResourceSpec.Name) - assert.Equal(t, "text/plain", textResourceSpec.ContentType) - assert.True(t, textResourceSpec.Compression) -} - func TestResolvePodTemplate(t *testing.T) { _, rootCmd, _ := initializeRunCmdOptions(t) templateText := ` diff --git a/pkg/util/camel/camel_dependencies.go b/pkg/util/camel/camel_dependencies.go index 509c4021af..c8dea7ca90 100644 --- a/pkg/util/camel/camel_dependencies.go +++ b/pkg/util/camel/camel_dependencies.go @@ -19,6 +19,7 @@ package camel import ( "fmt" + "io" "path" "path/filepath" "strings" @@ -29,7 +30,55 @@ import ( "github.com/rs/xid" ) -// ManageIntegrationDependencies --. +// NormalizeDependency converts different forms of camel dependencies +// -- `camel-xxx`, `camel-quarkus-xxx`, and `camel-quarkus:xxx` -- +// into the unified form `camel:xxx`. +func NormalizeDependency(dependency string) string { + newDep := dependency + switch { + case strings.HasPrefix(newDep, "camel-quarkus-"): + newDep = "camel:" + strings.TrimPrefix(dependency, "camel-quarkus-") + case strings.HasPrefix(newDep, "camel-quarkus:"): + newDep = "camel:" + strings.TrimPrefix(dependency, "camel-quarkus:") + case strings.HasPrefix(newDep, "camel-k-"): + newDep = "camel-k:" + strings.TrimPrefix(dependency, "camel-k-") + case strings.HasPrefix(newDep, "camel-k:"): + // do nothing + case strings.HasPrefix(newDep, "camel-"): + newDep = "camel:" + strings.TrimPrefix(dependency, "camel-") + } + return newDep +} + +type Output interface { + OutOrStdout() io.Writer + ErrOrStderr() io.Writer +} + +// ValidateDependencies validates dependencies against Camel catalog. +// It only shows warning and does not throw error in case the Catalog is just not complete +// and we don't want to let it stop the process. +func ValidateDependencies(catalog *RuntimeCatalog, dependencies []string, out Output) { + for _, d := range dependencies { + ValidateDependency(catalog, d, out) + } +} + +// ValidateDependency validates a dependency against Camel catalog. +// It only shows warning and does not throw error in case the Catalog is just not complete +// and we don't want to let it stop the process. +func ValidateDependency(catalog *RuntimeCatalog, dependency string, out Output) { + if !strings.HasPrefix(dependency, "camel:") { + return + } + + artifact := strings.TrimPrefix(dependency, "camel:") + if ok := catalog.HasArtifact(artifact); !ok { + fmt.Fprintf(out.ErrOrStderr(), "Warning: dependency %s not found in Camel catalog\n", dependency) + } +} + +// ManageIntegrationDependencies sets up all the required dependencies for the given Maven project. func ManageIntegrationDependencies(project *maven.Project, dependencies []string, catalog *RuntimeCatalog) error { // Add dependencies from build if err := addDependencies(project, dependencies, catalog); err != nil { diff --git a/pkg/util/camel/camel_dependencies_test.go b/pkg/util/camel/camel_dependencies_test.go new file mode 100644 index 0000000000..8bb78f3da6 --- /dev/null +++ b/pkg/util/camel/camel_dependencies_test.go @@ -0,0 +1,33 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package camel + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNormalizeDependency(t *testing.T) { + assert.Equal(t, "camel:file", NormalizeDependency("camel-file")) + assert.Equal(t, "camel:file", NormalizeDependency("camel:file")) + assert.Equal(t, "camel:file", NormalizeDependency("camel-quarkus-file")) + assert.Equal(t, "camel:file", NormalizeDependency("camel-quarkus:file")) + assert.Equal(t, "camel-k:knative", NormalizeDependency("camel-k-knative")) + assert.Equal(t, "camel-k:knative", NormalizeDependency("camel-k:knative")) +} From 2652eed135bf6eaee106301fb2d6861be44bbfd0 Mon Sep 17 00:00:00 2001 From: Tadayoshi Sato Date: Wed, 14 Sep 2022 12:36:07 +0900 Subject: [PATCH 06/11] chore(lint): lll linter fix for build_kit.go --- pkg/controller/integration/build_kit.go | 46 ++++++++++++++++++------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/pkg/controller/integration/build_kit.go b/pkg/controller/integration/build_kit.go index a08ff0bbaf..df568b71f9 100644 --- a/pkg/controller/integration/build_kit.go +++ b/pkg/controller/integration/build_kit.go @@ -64,16 +64,20 @@ func (action *buildKitAction) Handle(ctx context.Context, integration *v1.Integr // if integration.Status.IntegrationKit != nil { // IntegrationKit fully defined so find it - action.L.Debugf("Finding integration kit %s for integration %s\n", integration.Status.IntegrationKit.Name, integration.Name) - kit, err := kubernetes.GetIntegrationKit(ctx, action.client, integration.Status.IntegrationKit.Name, integration.Status.IntegrationKit.Namespace) + action.L.Debugf("Finding integration kit %s for integration %s\n", + integration.Status.IntegrationKit.Name, integration.Name) + kit, err := kubernetes.GetIntegrationKit(ctx, action.client, + integration.Status.IntegrationKit.Name, integration.Status.IntegrationKit.Namespace) if err != nil { - return nil, errors.Wrapf(err, "unable to find integration kit %s/%s, %s", integration.Status.IntegrationKit.Namespace, integration.Status.IntegrationKit.Name, err) + return nil, errors.Wrapf(err, "unable to find integration kit %s/%s, %s", + integration.Status.IntegrationKit.Namespace, integration.Status.IntegrationKit.Name, err) } if kit.Labels[v1.IntegrationKitTypeLabel] == v1.IntegrationKitTypePlatform { match, err := integrationMatches(integration, kit) if err != nil { - return nil, errors.Wrapf(err, "unable to match any integration kit with integration %s/%s", integration.Namespace, integration.Name) + return nil, errors.Wrapf(err, "unable to match any integration kit with integration %s/%s", + integration.Namespace, integration.Name) } else if !match { // We need to re-generate a kit, or search for a new one that // matches the integration, so let's remove the association @@ -82,7 +86,10 @@ func (action *buildKitAction) Handle(ctx context.Context, integration *v1.Integr // // All tests & conditionals check for a nil assignment // - action.L.Debug("No match found between integration and integrationkit. Resetting integration's integrationkit to empty", "integration", integration.Name, "integrationkit", integration.Status.IntegrationKit.Name, "namespace", integration.Namespace) + action.L.Debug("No match found between integration and integrationkit. Resetting integration's integrationkit to empty", + "integration", integration.Name, + "integrationkit", integration.Status.IntegrationKit.Name, + "namespace", integration.Namespace) integration.SetIntegrationKit(nil) return integration, nil } @@ -106,16 +113,21 @@ func (action *buildKitAction) Handle(ctx context.Context, integration *v1.Integr action.L.Debug("No kit specified in integration status so looking up", "integration", integration.Name, "namespace", integration.Namespace) existingKits, err := lookupKitsForIntegration(ctx, action.client, integration) if err != nil { - return nil, errors.Wrapf(err, "failed to lookup kits for integration %s/%s", integration.Namespace, integration.Name) + return nil, errors.Wrapf(err, "failed to lookup kits for integration %s/%s", + integration.Namespace, integration.Name) } - action.L.Debug("Applying traits to integration", "integration", integration.Name, "namespace", integration.Namespace) + action.L.Debug("Applying traits to integration", + "integration", integration.Name, + "namespace", integration.Namespace) env, err := trait.Apply(ctx, action.client, integration, nil) if err != nil { - return nil, errors.Wrapf(err, "failed to apply traits to integration %s/%s", integration.Namespace, integration.Name) + return nil, errors.Wrapf(err, "failed to apply traits to integration %s/%s", + integration.Namespace, integration.Name) } - action.L.Debug("Searching integration kits to assign to integration", "integration", integration.Name, "namespace", integration.Namespace) + action.L.Debug("Searching integration kits to assign to integration", "integration", + integration.Name, "namespace", integration.Namespace) var integrationKit *v1.IntegrationKit kits: for _, kit := range env.IntegrationKits { @@ -127,7 +139,9 @@ kits: action.L.Debug("Comparing existing kit with environment", "env kit", kit.Name, "existing kit", k.Name) match, err := kitMatches(&kit, k) if err != nil { - return nil, errors.Wrapf(err, "error occurred matches integration kits with environment for integration %s/%s", integration.Namespace, integration.Name) + return nil, errors.Wrapf(err, + "error occurred matches integration kits with environment for integration %s/%s", + integration.Namespace, integration.Name) } if match { if integrationKit == nil || @@ -143,9 +157,13 @@ kits: } } - action.L.Debug("No existing kit available for integration. Creating a new one.", "integration", integration.Name, "namespace", integration.Namespace, "integration kit", kit.Name) + action.L.Debug("No existing kit available for integration. Creating a new one.", + "integration", integration.Name, + "namespace", integration.Namespace, + "integration kit", kit.Name) if err := action.client.Create(ctx, &kit); err != nil { - return nil, errors.Wrapf(err, "failed to create new integration kit for integration %s/%s", integration.Namespace, integration.Name) + return nil, errors.Wrapf(err, "failed to create new integration kit for integration %s/%s", + integration.Namespace, integration.Name) } if integrationKit == nil { integrationKit = &kit @@ -162,7 +180,9 @@ kits: integration.Status.Phase = v1.IntegrationPhaseDeploying } } else { - action.L.Debug("Not yet able to assign an integration kit to integration", "integration", integration.Name, "namespace", integration.Namespace) + action.L.Debug("Not yet able to assign an integration kit to integration", + "integration", integration.Name, + "namespace", integration.Namespace) } return integration, nil From 630816b265faf3933d493151af31c7dc660b8874 Mon Sep 17 00:00:00 2001 From: Tadayoshi Sato Date: Wed, 14 Sep 2022 13:14:34 +0900 Subject: [PATCH 07/11] feat(controller): log error if a dependency is not valid in catalog --- pkg/trait/dependencies.go | 4 ++++ pkg/util/camel/camel_dependencies.go | 17 +++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/pkg/trait/dependencies.go b/pkg/trait/dependencies.go index 5a68c3cefc..41f1f3d612 100644 --- a/pkg/trait/dependencies.go +++ b/pkg/trait/dependencies.go @@ -26,6 +26,7 @@ import ( traitv1 "github.com/apache/camel-k/pkg/apis/camel/v1/trait" "github.com/apache/camel-k/pkg/metadata" "github.com/apache/camel-k/pkg/util" + "github.com/apache/camel-k/pkg/util/camel" "github.com/apache/camel-k/pkg/util/kubernetes" ) @@ -56,6 +57,9 @@ func (t *dependenciesTrait) Apply(e *Environment) error { dependencies := strset.New() if e.Integration.Spec.Dependencies != nil { + if err := camel.ValidateDependenciesE(e.CamelCatalog, e.Integration.Spec.Dependencies); err != nil { + return err + } dependencies.Add(e.Integration.Spec.Dependencies...) } diff --git a/pkg/util/camel/camel_dependencies.go b/pkg/util/camel/camel_dependencies.go index c8dea7ca90..29db9f83f2 100644 --- a/pkg/util/camel/camel_dependencies.go +++ b/pkg/util/camel/camel_dependencies.go @@ -78,6 +78,23 @@ func ValidateDependency(catalog *RuntimeCatalog, dependency string, out Output) } } +// ValidateDependenciesE validates dependencies against Camel catalog and throws error +// if it doesn't exist in the catalog. +func ValidateDependenciesE(catalog *RuntimeCatalog, dependencies []string) error { + for _, dependency := range dependencies { + if !strings.HasPrefix(dependency, "camel:") { + continue + } + + artifact := strings.TrimPrefix(dependency, "camel:") + if ok := catalog.HasArtifact(artifact); !ok { + return fmt.Errorf("dependency %s not found in Camel catalog", dependency) + } + } + + return nil +} + // ManageIntegrationDependencies sets up all the required dependencies for the given Maven project. func ManageIntegrationDependencies(project *maven.Project, dependencies []string, catalog *RuntimeCatalog) error { // Add dependencies from build From ce0ae6e5c239a9c2c0e3559395ef6d335f9bac7c Mon Sep 17 00:00:00 2001 From: Tadayoshi Sato Date: Wed, 14 Sep 2022 13:41:18 +0900 Subject: [PATCH 08/11] feat(cli): show warn message when Camel dependencies specified as Maven GAV Such as: - mvn:org.apache.camel:camel-log:3.18.0 - mvn:org.apache.camel-quarkus:camel-quarkus-log:2.11.0 --- pkg/util/camel/camel_dependencies.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/pkg/util/camel/camel_dependencies.go b/pkg/util/camel/camel_dependencies.go index 29db9f83f2..5bf7ad517a 100644 --- a/pkg/util/camel/camel_dependencies.go +++ b/pkg/util/camel/camel_dependencies.go @@ -68,14 +68,22 @@ func ValidateDependencies(catalog *RuntimeCatalog, dependencies []string, out Ou // It only shows warning and does not throw error in case the Catalog is just not complete // and we don't want to let it stop the process. func ValidateDependency(catalog *RuntimeCatalog, dependency string, out Output) { - if !strings.HasPrefix(dependency, "camel:") { - return + switch { + case strings.HasPrefix(dependency, "camel:"): + artifact := strings.TrimPrefix(dependency, "camel:") + if ok := catalog.HasArtifact(artifact); !ok { + fmt.Fprintf(out.ErrOrStderr(), "Warning: dependency %s not found in Camel catalog\n", dependency) + } + case strings.HasPrefix(dependency, "mvn:org.apache.camel:"): + component := strings.Split(dependency, ":")[2] + fmt.Fprintf(out.ErrOrStderr(), "Warning: do not use %s. Use %s instead\n", + dependency, NormalizeDependency(component)) + case strings.HasPrefix(dependency, "mvn:org.apache.camel.quarkus:"): + component := strings.Split(dependency, ":")[2] + fmt.Fprintf(out.ErrOrStderr(), "Warning: do not use %s. Use %s instead\n", + dependency, NormalizeDependency(component)) } - artifact := strings.TrimPrefix(dependency, "camel:") - if ok := catalog.HasArtifact(artifact); !ok { - fmt.Fprintf(out.ErrOrStderr(), "Warning: dependency %s not found in Camel catalog\n", dependency) - } } // ValidateDependenciesE validates dependencies against Camel catalog and throws error From 305dfe8744c0160e5b9aaba09c037bce4a4836ab Mon Sep 17 00:00:00 2001 From: Tadayoshi Sato Date: Wed, 14 Sep 2022 14:04:46 +0900 Subject: [PATCH 09/11] test(e2e): add E2E tests for handling of invalid components and dependencies --- e2e/global/common/integration_fail_test.go | 29 +++++++++++++++++++++- e2e/local/local_build_test.go | 15 ++++++++--- e2e/local/local_inspect_test.go | 16 +++++++++--- e2e/local/local_run_test.go | 15 ++++++++--- 4 files changed, 62 insertions(+), 13 deletions(-) diff --git a/e2e/global/common/integration_fail_test.go b/e2e/global/common/integration_fail_test.go index cf999630cb..ebfeda9f5d 100644 --- a/e2e/global/common/integration_fail_test.go +++ b/e2e/global/common/integration_fail_test.go @@ -81,11 +81,38 @@ func TestBadRouteIntegration(t *testing.T) { Expect(Kamel("delete", "--all", "-n", ns).Execute()).To(Succeed()) }) + t.Run("run invalid dependency java route", func(t *testing.T) { + RegisterTestingT(t) + name := "invalid-dependency" + Expect(KamelRunWithID(operatorID, ns, "files/Java.java", "--name", name, + "-d", "camel:non-existent").Execute()).To(Succeed()) + // Integration in error with Initialization Failed condition + Eventually(IntegrationPhase(ns, name), TestTimeoutLong).Should(Equal(v1.IntegrationPhaseError)) + Eventually(IntegrationConditionStatus(ns, name, v1.IntegrationConditionReady), TestTimeoutShort). + Should(Equal(corev1.ConditionFalse)) + Eventually(IntegrationCondition(ns, name, v1.IntegrationConditionReady), TestTimeoutShort).Should(And( + WithTransform(IntegrationConditionReason, Equal(v1.IntegrationConditionInitializationFailedReason)), + WithTransform(IntegrationConditionMessage, HavePrefix("error during trait customization")), + )) + // Kit shouldn't be created + Consistently(IntegrationKit(ns, name), 10*time.Second).Should(BeEmpty()) + + // Fixing the route should reconcile the Integration in Initialization Failed condition to Running + Expect(KamelRunWithID(operatorID, ns, "files/Java.java", "--name", name).Execute()).To(Succeed()) + Eventually(IntegrationPodPhase(ns, name), TestTimeoutLong).Should(Equal(corev1.PodRunning)) + Eventually(IntegrationConditionStatus(ns, name, v1.IntegrationConditionReady), TestTimeoutShort). + Should(Equal(corev1.ConditionTrue)) + Eventually(IntegrationLogs(ns, name), TestTimeoutShort).Should(ContainSubstring("Magicstring!")) + + // Clean up + Expect(Kamel("delete", "--all", "-n", ns).Execute()).To(Succeed()) + }) + t.Run("run unresolvable component java route", func(t *testing.T) { RegisterTestingT(t) name := "unresolvable-route" Expect(KamelRunWithID(operatorID, ns, "files/Unresolvable.java", "--name", name).Execute()).To(Succeed()) - // Integration in error + // Integration in error with Initialization Failed condition Eventually(IntegrationPhase(ns, name), TestTimeoutShort).Should(Equal(v1.IntegrationPhaseError)) Eventually(IntegrationConditionStatus(ns, name, v1.IntegrationConditionReady), TestTimeoutShort). Should(Equal(corev1.ConditionFalse)) diff --git a/e2e/local/local_build_test.go b/e2e/local/local_build_test.go index 54d9be9a6d..543f0da2c5 100644 --- a/e2e/local/local_build_test.go +++ b/e2e/local/local_build_test.go @@ -126,11 +126,17 @@ func TestLocalBuildWithInvalidDependency(t *testing.T) { file := testutil.MakeTempCopy(t, "files/yaml.yaml") image := "test/test-" + strings.ToLower(util.RandomString(10)) - kamelBuild := KamelWithContext(ctx, "local", "build", file, "--image", image, "-d", "camel-xxx") + kamelBuild := KamelWithContext(ctx, "local", "build", file, "--image", image, + "-d", "camel-xxx", + "-d", "mvn:org.apache.camel:camel-http:3.18.0", + "-d", "mvn:org.apache.camel.quarkus:camel-quarkus-netty:2.11.0") kamelBuild.SetOut(pipew) kamelBuild.SetErr(pipew) - logScanner := testutil.NewLogScanner(ctx, piper, "Warning: dependency camel:xxx not found in Camel catalog") + warn1 := "Warning: dependency camel:xxx not found in Camel catalog" + warn2 := "Warning: do not use mvn:org.apache.camel:camel-http:3.18.0. Use camel:http instead" + warn3 := "Warning: do not use mvn:org.apache.camel.quarkus:camel-quarkus-netty:2.11.0. Use camel:netty instead" + logScanner := testutil.NewLogScanner(ctx, piper, warn1, warn2, warn3) var wg sync.WaitGroup wg.Add(1) @@ -141,8 +147,9 @@ func TestLocalBuildWithInvalidDependency(t *testing.T) { cancel() }() - Eventually(logScanner.IsFound("Warning: dependency camel:xxx not found in Camel catalog"), TestTimeoutShort). - Should(BeTrue()) + Eventually(logScanner.IsFound(warn1), TestTimeoutShort).Should(BeTrue()) + Eventually(logScanner.IsFound(warn2), TestTimeoutShort).Should(BeTrue()) + Eventually(logScanner.IsFound(warn3), TestTimeoutShort).Should(BeTrue()) wg.Wait() } diff --git a/e2e/local/local_inspect_test.go b/e2e/local/local_inspect_test.go index 92dbec905f..20aad1f9b7 100644 --- a/e2e/local/local_inspect_test.go +++ b/e2e/local/local_inspect_test.go @@ -76,12 +76,19 @@ func TestLocalInspectWithDependencies(t *testing.T) { file := testutil.MakeTempCopy(t, "files/yaml.yaml") - kamelInspect := KamelWithContext(ctx, "local", "inspect", file, "-d", "camel-amqp", "-d", "camel-xxx") + kamelInspect := KamelWithContext(ctx, "local", "inspect", file, + "-d", "camel-amqp", + "-d", "camel-xxx", + "-d", "mvn:org.apache.camel:camel-http:3.18.0", + "-d", "mvn:org.apache.camel.quarkus:camel-quarkus-netty:2.11.0") kamelInspect.SetOut(pipew) kamelInspect.SetErr(pipew) + warn1 := "Warning: dependency camel:xxx not found in Camel catalog" + warn2 := "Warning: do not use mvn:org.apache.camel:camel-http:3.18.0. Use camel:http instead" + warn3 := "Warning: do not use mvn:org.apache.camel.quarkus:camel-quarkus-netty:2.11.0. Use camel:netty instead" logScanner := testutil.NewLogScanner(ctx, piper, - "Warning: dependency camel:xxx not found in Camel catalog", + warn1, warn2, warn3, "camel:amqp", "camel:log", "camel:timer", @@ -93,8 +100,9 @@ func TestLocalInspectWithDependencies(t *testing.T) { cancel() }() - Eventually(logScanner.IsFound("Warning: dependency camel:xxx not found in Camel catalog"), TestTimeoutShort). - Should(BeTrue()) + Eventually(logScanner.IsFound(warn1), TestTimeoutShort).Should(BeTrue()) + Eventually(logScanner.IsFound(warn2), TestTimeoutShort).Should(BeTrue()) + Eventually(logScanner.IsFound(warn3), TestTimeoutShort).Should(BeTrue()) Eventually(logScanner.IsFound("camel:amqp"), TestTimeoutShort).Should(BeTrue()) Eventually(logScanner.IsFound("camel:log"), TestTimeoutShort).Should(BeTrue()) Eventually(logScanner.IsFound("camel:timer"), TestTimeoutShort).Should(BeTrue()) diff --git a/e2e/local/local_run_test.go b/e2e/local/local_run_test.go index 3c3dbd49e6..9ac6f864b4 100644 --- a/e2e/local/local_run_test.go +++ b/e2e/local/local_run_test.go @@ -96,11 +96,17 @@ func TestLocalRunWithInvalidDependency(t *testing.T) { file := testutil.MakeTempCopy(t, "files/yaml.yaml") - kamelRun := KamelWithContext(ctx, "local", "run", file, "-d", "camel-xxx") + kamelRun := KamelWithContext(ctx, "local", "run", file, + "-d", "camel-xxx", + "-d", "mvn:org.apache.camel:camel-http:3.18.0", + "-d", "mvn:org.apache.camel.quarkus:camel-quarkus-netty:2.11.0") kamelRun.SetOut(pipew) kamelRun.SetErr(pipew) - logScanner := testutil.NewLogScanner(ctx, piper, "Warning: dependency camel:xxx not found in Camel catalog") + warn1 := "Warning: dependency camel:xxx not found in Camel catalog" + warn2 := "Warning: do not use mvn:org.apache.camel:camel-http:3.18.0. Use camel:http instead" + warn3 := "Warning: do not use mvn:org.apache.camel.quarkus:camel-quarkus-netty:2.11.0. Use camel:netty instead" + logScanner := testutil.NewLogScanner(ctx, piper, warn1, warn2, warn3) var wg sync.WaitGroup wg.Add(1) @@ -111,8 +117,9 @@ func TestLocalRunWithInvalidDependency(t *testing.T) { cancel() }() - Eventually(logScanner.IsFound("Warning: dependency camel:xxx not found in Camel catalog"), TestTimeoutShort). - Should(BeTrue()) + Eventually(logScanner.IsFound(warn1), TestTimeoutShort).Should(BeTrue()) + Eventually(logScanner.IsFound(warn2), TestTimeoutShort).Should(BeTrue()) + Eventually(logScanner.IsFound(warn3), TestTimeoutShort).Should(BeTrue()) wg.Wait() } From bc531cc5cfc2f63206889e71b0f6b49255559a02 Mon Sep 17 00:00:00 2001 From: Tadayoshi Sato Date: Thu, 15 Sep 2022 15:17:40 +0900 Subject: [PATCH 10/11] fix(e2e): workaround for kamel local inspect e2e flakiness --- e2e/local/local_inspect_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/e2e/local/local_inspect_test.go b/e2e/local/local_inspect_test.go index 20aad1f9b7..902eac973a 100644 --- a/e2e/local/local_inspect_test.go +++ b/e2e/local/local_inspect_test.go @@ -50,7 +50,7 @@ func TestLocalInspect(t *testing.T) { logScanner := testutil.NewLogScanner(ctx, piper, "camel:log", "camel:timer", - "mvn:org.apache.camel.quarkus:camel-quarkus-yaml-dsl", + //"mvn:org.apache.camel.quarkus:camel-quarkus-yaml-dsl", ) go func() { @@ -61,8 +61,7 @@ func TestLocalInspect(t *testing.T) { Eventually(logScanner.IsFound("camel:log"), TestTimeoutShort).Should(BeTrue()) Eventually(logScanner.IsFound("camel:timer"), TestTimeoutShort).Should(BeTrue()) - Eventually(logScanner.IsFound("mvn:org.apache.camel.quarkus:camel-quarkus-yaml-dsl"), TestTimeoutShort). - Should(BeTrue()) + //Eventually(logScanner.IsFound("mvn:org.apache.camel.quarkus:camel-quarkus-yaml-dsl"), TestTimeoutShort).Should(BeTrue()) } func TestLocalInspectWithDependencies(t *testing.T) { From ce25a7441e48db39b69bf58fef2c58a235d8c515 Mon Sep 17 00:00:00 2001 From: Tadayoshi Sato Date: Thu, 15 Sep 2022 00:01:48 +0900 Subject: [PATCH 11/11] fix(controller): dependency check should also look for loader artifacts in catalog --- pkg/cmd/completion_bash.go | 7 ++- pkg/util/camel/camel_dependencies.go | 4 +- pkg/util/camel/camel_runtime_catalog.go | 31 ++++++++++++- pkg/util/camel/camel_runtime_catalog_test.go | 48 ++++++++++++++++++++ 4 files changed, 85 insertions(+), 5 deletions(-) create mode 100644 pkg/util/camel/camel_runtime_catalog_test.go diff --git a/pkg/cmd/completion_bash.go b/pkg/cmd/completion_bash.go index 2b04ccec2b..66654c0251 100644 --- a/pkg/cmd/completion_bash.go +++ b/pkg/cmd/completion_bash.go @@ -297,13 +297,18 @@ func computeCamelDependencies() string { catalog = camel.NewRuntimeCatalog(v1.CamelCatalog{}.Spec) } - results := make([]string, 0, len(catalog.Artifacts)) + results := make([]string, 0, len(catalog.Artifacts)+len(catalog.Loaders)) for a := range catalog.Artifacts { // skipping camel-k-* and other artifacts as they may not be useful for cli completion if strings.HasPrefix(a, "camel-quarkus-") { results = append(results, camel.NormalizeDependency(a)) } } + for _, l := range catalog.Loaders { + if strings.HasPrefix(l.ArtifactID, "camel-quarkus-") { + results = append(results, camel.NormalizeDependency(l.ArtifactID)) + } + } sort.Strings(results) return strings.Join(results, " ") diff --git a/pkg/util/camel/camel_dependencies.go b/pkg/util/camel/camel_dependencies.go index 5bf7ad517a..9bee5f7026 100644 --- a/pkg/util/camel/camel_dependencies.go +++ b/pkg/util/camel/camel_dependencies.go @@ -71,7 +71,7 @@ func ValidateDependency(catalog *RuntimeCatalog, dependency string, out Output) switch { case strings.HasPrefix(dependency, "camel:"): artifact := strings.TrimPrefix(dependency, "camel:") - if ok := catalog.HasArtifact(artifact); !ok { + if ok := catalog.IsValidArtifact(artifact); !ok { fmt.Fprintf(out.ErrOrStderr(), "Warning: dependency %s not found in Camel catalog\n", dependency) } case strings.HasPrefix(dependency, "mvn:org.apache.camel:"): @@ -95,7 +95,7 @@ func ValidateDependenciesE(catalog *RuntimeCatalog, dependencies []string) error } artifact := strings.TrimPrefix(dependency, "camel:") - if ok := catalog.HasArtifact(artifact); !ok { + if ok := catalog.IsValidArtifact(artifact); !ok { return fmt.Errorf("dependency %s not found in Camel catalog", dependency) } } diff --git a/pkg/util/camel/camel_runtime_catalog.go b/pkg/util/camel/camel_runtime_catalog.go index f3a6b37592..25da254e4c 100644 --- a/pkg/util/camel/camel_runtime_catalog.go +++ b/pkg/util/camel/camel_runtime_catalog.go @@ -23,7 +23,7 @@ import ( v1 "github.com/apache/camel-k/pkg/apis/camel/v1" ) -// NewRuntimeCatalog --. +// NewRuntimeCatalog creates a runtime catalog with the given catalog spec. func NewRuntimeCatalog(spec v1.CamelCatalogSpec) *RuntimeCatalog { catalog := RuntimeCatalog{} catalog.CamelCatalogSpec = spec @@ -32,6 +32,7 @@ func NewRuntimeCatalog(spec v1.CamelCatalogSpec) *RuntimeCatalog { catalog.schemesByID = make(map[string]v1.CamelScheme) catalog.languageDependencies = make(map[string]string) catalog.javaTypeDependencies = make(map[string]string) + catalog.loaderByArtifact = make(map[string]string) for id, artifact := range catalog.Artifacts { for _, scheme := range artifact.Schemes { @@ -66,10 +67,14 @@ func NewRuntimeCatalog(spec v1.CamelCatalogSpec) *RuntimeCatalog { } } + for id, loader := range catalog.Loaders { + catalog.loaderByArtifact[loader.ArtifactID] = id + } + return &catalog } -// RuntimeCatalog --. +// RuntimeCatalog represents the data structure for a runtime catalog. type RuntimeCatalog struct { v1.CamelCatalogSpec @@ -78,6 +83,7 @@ type RuntimeCatalog struct { schemesByID map[string]v1.CamelScheme languageDependencies map[string]string javaTypeDependencies map[string]string + loaderByArtifact map[string]string } // HasArtifact checks if the given artifact is present in the catalog. @@ -96,6 +102,27 @@ func (c *RuntimeCatalog) HasArtifact(artifact string) bool { return ok } +// HasLoaderByArtifact checks if the given artifact is a loader in the catalog. +func (c *RuntimeCatalog) HasLoaderByArtifact(artifact string) bool { + a := artifact + if !strings.HasPrefix(a, "camel-") { + if c.Runtime.Provider == v1.RuntimeProviderQuarkus { + a = "camel-quarkus-" + a + } else { + a = "camel-" + a + } + } + + _, ok := c.loaderByArtifact[a] + + return ok +} + +// IsValidArtifact returns true if the given artifact is an artifact or loader in the catalog. +func (c *RuntimeCatalog) IsValidArtifact(artifact string) bool { + return c.HasArtifact(artifact) || c.HasLoaderByArtifact(artifact) +} + // GetArtifactByScheme returns the artifact corresponding to the given component scheme. func (c *RuntimeCatalog) GetArtifactByScheme(scheme string) *v1.CamelArtifact { if id, ok := c.artifactByScheme[scheme]; ok { diff --git a/pkg/util/camel/camel_runtime_catalog_test.go b/pkg/util/camel/camel_runtime_catalog_test.go new file mode 100644 index 0000000000..487547a0a8 --- /dev/null +++ b/pkg/util/camel/camel_runtime_catalog_test.go @@ -0,0 +1,48 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package camel + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoaderByArtifact(t *testing.T) { + catalog, err := DefaultCatalog() + require.NoError(t, err) + assert.Equal(t, "groovy", catalog.loaderByArtifact["camel-quarkus-groovy-dsl"]) + assert.Equal(t, "java", catalog.loaderByArtifact["camel-quarkus-java-joor-dsl"]) + assert.Equal(t, "js", catalog.loaderByArtifact["camel-quarkus-js-dsl"]) + assert.Equal(t, "kts", catalog.loaderByArtifact["camel-quarkus-kotlin-dsl"]) + assert.Equal(t, "xml", catalog.loaderByArtifact["camel-quarkus-xml-io-dsl"]) + assert.Equal(t, "yaml", catalog.loaderByArtifact["camel-quarkus-yaml-dsl"]) +} + +func TestHasLoaderByArtifact(t *testing.T) { + catalog, err := DefaultCatalog() + require.NoError(t, err) + assert.True(t, catalog.HasLoaderByArtifact("groovy-dsl")) + assert.True(t, catalog.HasLoaderByArtifact("java-joor-dsl")) + assert.True(t, catalog.HasLoaderByArtifact("js-dsl")) + assert.True(t, catalog.HasLoaderByArtifact("kotlin-dsl")) + assert.True(t, catalog.HasLoaderByArtifact("xml-io-dsl")) + assert.True(t, catalog.HasLoaderByArtifact("yaml-dsl")) + assert.False(t, catalog.HasLoaderByArtifact("python-dsl")) +}