diff --git a/cmd/skaffold/app/cmd/generate_pipeline.go b/cmd/skaffold/app/cmd/generate_pipeline.go index afbdcdb2e20..ff802f11517 100644 --- a/cmd/skaffold/app/cmd/generate_pipeline.go +++ b/cmd/skaffold/app/cmd/generate_pipeline.go @@ -29,18 +29,24 @@ import ( "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest" ) +var ( + configFiles []string +) + func NewCmdGeneratePipeline() *cobra.Command { return NewCmd("generate-pipeline"). Hidden(). WithDescription("[ALPHA] Generate tekton pipeline from skaffold.yaml"). WithCommonFlags(). - WithFlags(func(f *pflag.FlagSet) {}). + WithFlags(func(f *pflag.FlagSet) { + f.StringSliceVar(&configFiles, "config-files", nil, "Select additional files whose artifacts to use when generating pipeline.") + }). NoArgs(cancelWithCtrlC(context.Background(), doGeneratePipeline)) } func doGeneratePipeline(ctx context.Context, out io.Writer) error { return withRunner(ctx, func(r runner.Runner, config *latest.SkaffoldConfig) error { - if err := r.GeneratePipeline(ctx, out, config, "pipeline.yaml"); err != nil { + if err := r.GeneratePipeline(ctx, out, config, configFiles, "pipeline.yaml"); err != nil { return errors.Wrap(err, "generating ") } color.Default.Fprintln(out, "Pipeline config written to pipeline.yaml!") diff --git a/integration/generate_pipeline_test.go b/integration/generate_pipeline_test.go index 3543e9b8b69..8aa8c922ea3 100644 --- a/integration/generate_pipeline_test.go +++ b/integration/generate_pipeline_test.go @@ -19,11 +19,17 @@ package integration import ( "bytes" "io/ioutil" + "os" "testing" "github.com/GoogleContainerTools/skaffold/integration/skaffold" ) +type configContents struct { + path string + data []byte +} + func TestGeneratePipeline(t *testing.T) { if testing.Short() { t.Skip("skipping integration test") @@ -35,38 +41,54 @@ func TestGeneratePipeline(t *testing.T) { tests := []struct { description string dir string - responses []byte + input []byte + args []string + configFiles []string }{ { description: "no profiles", dir: "testdata/generate_pipeline/no_profiles", - responses: []byte("y"), + input: []byte("y\n"), }, { description: "existing oncluster profile", dir: "testdata/generate_pipeline/existing_oncluster", - responses: []byte(""), + input: []byte(""), }, { description: "existing other profile", dir: "testdata/generate_pipeline/existing_other", - responses: []byte("y"), + input: []byte("y\n"), + }, + { + description: "multiple skaffold.yamls to create pipeline from", + dir: "testdata/generate_pipeline/multiple_configs", + input: []byte{'y', '\n', 'y', '\n'}, + configFiles: []string{"sub-app/skaffold.yaml"}, }, } for _, test := range tests { t.Run(test.description, func(t *testing.T) { + + args, contents, err := getOriginalContents(test.args, test.dir, test.configFiles) + if err != nil { + t.Fatal(err) + } + defer writeOriginalContents(contents) + originalConfig, err := ioutil.ReadFile(test.dir + "/skaffold.yaml") if err != nil { t.Error("error reading skaffold yaml") } defer ioutil.WriteFile(test.dir+"/skaffold.yaml", originalConfig, 0755) + defer os.Remove(test.dir + "/pipeline.yaml") skaffoldEnv := []string{ "PIPELINE_GIT_URL=this-is-a-test", "PIPELINE_SKAFFOLD_VERSION=test-version", } - skaffold.GeneratePipeline().WithStdin([]byte("y\n")).WithEnv(skaffoldEnv).InDir(test.dir).RunOrFail(t) + skaffold.GeneratePipeline(args...).WithStdin(test.input).WithEnv(skaffoldEnv).InDir(test.dir).RunOrFail(t) checkFileContents(t, test.dir+"/expectedSkaffold.yaml", test.dir+"/skaffold.yaml") checkFileContents(t, test.dir+"/expectedPipeline.yaml", test.dir+"/pipeline.yaml") @@ -74,6 +96,30 @@ func TestGeneratePipeline(t *testing.T) { } } +func getOriginalContents(testArgs []string, testDir string, configFiles []string) ([]string, []configContents, error) { + var originalConfigs []configContents + if len(configFiles) != 0 { + for _, configFile := range configFiles { + testArgs = append(testArgs, []string{"--config-files", configFile}...) + + path := testDir + "/" + configFile + contents, err := ioutil.ReadFile(path) + if err != nil { + return nil, nil, err + } + originalConfigs = append(originalConfigs, configContents{path, contents}) + } + } + + return testArgs, originalConfigs, nil +} + +func writeOriginalContents(contents []configContents) { + for _, content := range contents { + ioutil.WriteFile(content.path, content.data, 0755) + } +} + func checkFileContents(t *testing.T, wantFile, gotFile string) { wantContents, err := ioutil.ReadFile(wantFile) if err != nil { diff --git a/integration/testdata/generate_pipeline/existing_oncluster/expectedPipeline.yaml b/integration/testdata/generate_pipeline/existing_oncluster/expectedPipeline.yaml index 5455e0eaef3..f00b440d981 100644 --- a/integration/testdata/generate_pipeline/existing_oncluster/expectedPipeline.yaml +++ b/integration/testdata/generate_pipeline/existing_oncluster/expectedPipeline.yaml @@ -14,7 +14,7 @@ apiVersion: tekton.dev/v1alpha1 kind: Task metadata: creationTimestamp: null - name: skaffold-build + name: skaffold-build-0 spec: inputs: resources: @@ -58,7 +58,7 @@ apiVersion: tekton.dev/v1alpha1 kind: Task metadata: creationTimestamp: null - name: skaffold-deploy + name: skaffold-deploy-0 spec: inputs: resources: @@ -90,7 +90,7 @@ spec: - name: source-repo type: git tasks: - - name: skaffold-build-task + - name: skaffold-build-0-task resources: inputs: - name: source @@ -99,15 +99,15 @@ spec: - name: source resource: source-repo taskRef: - name: skaffold-build - - name: skaffold-deploy-task + name: skaffold-build-0 + - name: skaffold-deploy-0-task resources: inputs: - from: - - skaffold-build-task + - skaffold-build-0-task name: source resource: source-repo taskRef: - name: skaffold-deploy + name: skaffold-deploy-0 status: {} --- diff --git a/integration/testdata/generate_pipeline/existing_other/expectedPipeline.yaml b/integration/testdata/generate_pipeline/existing_other/expectedPipeline.yaml index 5455e0eaef3..f00b440d981 100644 --- a/integration/testdata/generate_pipeline/existing_other/expectedPipeline.yaml +++ b/integration/testdata/generate_pipeline/existing_other/expectedPipeline.yaml @@ -14,7 +14,7 @@ apiVersion: tekton.dev/v1alpha1 kind: Task metadata: creationTimestamp: null - name: skaffold-build + name: skaffold-build-0 spec: inputs: resources: @@ -58,7 +58,7 @@ apiVersion: tekton.dev/v1alpha1 kind: Task metadata: creationTimestamp: null - name: skaffold-deploy + name: skaffold-deploy-0 spec: inputs: resources: @@ -90,7 +90,7 @@ spec: - name: source-repo type: git tasks: - - name: skaffold-build-task + - name: skaffold-build-0-task resources: inputs: - name: source @@ -99,15 +99,15 @@ spec: - name: source resource: source-repo taskRef: - name: skaffold-build - - name: skaffold-deploy-task + name: skaffold-build-0 + - name: skaffold-deploy-0-task resources: inputs: - from: - - skaffold-build-task + - skaffold-build-0-task name: source resource: source-repo taskRef: - name: skaffold-deploy + name: skaffold-deploy-0 status: {} --- diff --git a/integration/testdata/generate_pipeline/multiple_configs/expectedPipeline.yaml b/integration/testdata/generate_pipeline/multiple_configs/expectedPipeline.yaml new file mode 100644 index 00000000000..5955149ca52 --- /dev/null +++ b/integration/testdata/generate_pipeline/multiple_configs/expectedPipeline.yaml @@ -0,0 +1,202 @@ +apiVersion: tekton.dev/v1alpha1 +kind: PipelineResource +metadata: + creationTimestamp: null + name: source-git +spec: + params: + - name: url + value: this-is-a-test + type: git +status: {} +--- +apiVersion: tekton.dev/v1alpha1 +kind: Task +metadata: + creationTimestamp: null + name: skaffold-build-0 +spec: + inputs: + resources: + - name: source + outputImageDir: "" + targetPath: "" + type: git + outputs: + resources: + - name: source + outputImageDir: "" + targetPath: "" + type: git + steps: + - args: + - --filename + - skaffold.yaml + - --profile + - oncluster + - --file-output + - build.out + command: + - skaffold + - build + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /secret/kaniko-secret + image: gcr.io/k8s-skaffold/skaffold:test-version + name: run-build + resources: {} + volumeMounts: + - mountPath: /secret + name: kaniko-secret + workingDir: /workspace/source + volumes: + - name: kaniko-secret + secret: + secretName: kaniko-secret +--- +apiVersion: tekton.dev/v1alpha1 +kind: Task +metadata: + creationTimestamp: null + name: skaffold-build-1 +spec: + inputs: + resources: + - name: source + outputImageDir: "" + targetPath: "" + type: git + outputs: + resources: + - name: source + outputImageDir: "" + targetPath: "" + type: git + steps: + - args: + - --filename + - sub-app/skaffold.yaml + - --profile + - oncluster + - --file-output + - build.out + command: + - skaffold + - build + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /secret/kaniko-secret + image: gcr.io/k8s-skaffold/skaffold:test-version + name: run-build + resources: {} + volumeMounts: + - mountPath: /secret + name: kaniko-secret + workingDir: /workspace/source + volumes: + - name: kaniko-secret + secret: + secretName: kaniko-secret +--- +apiVersion: tekton.dev/v1alpha1 +kind: Task +metadata: + creationTimestamp: null + name: skaffold-deploy-0 +spec: + inputs: + resources: + - name: source + outputImageDir: "" + targetPath: "" + type: git + steps: + - args: + - --filename + - skaffold.yaml + - --build-artifacts + - build.out + command: + - skaffold + - deploy + image: gcr.io/k8s-skaffold/skaffold:test-version + name: run-deploy + resources: {} + workingDir: /workspace/source +--- +apiVersion: tekton.dev/v1alpha1 +kind: Task +metadata: + creationTimestamp: null + name: skaffold-deploy-1 +spec: + inputs: + resources: + - name: source + outputImageDir: "" + targetPath: "" + type: git + steps: + - args: + - --filename + - sub-app/skaffold.yaml + - --build-artifacts + - build.out + command: + - skaffold + - deploy + image: gcr.io/k8s-skaffold/skaffold:test-version + name: run-deploy + resources: {} + workingDir: /workspace/source +--- +apiVersion: tekton.dev/v1alpha1 +kind: Pipeline +metadata: + creationTimestamp: null + name: skaffold-pipeline +spec: + resources: + - name: source-repo + type: git + tasks: + - name: skaffold-build-0-task + resources: + inputs: + - name: source + resource: source-repo + outputs: + - name: source + resource: source-repo + taskRef: + name: skaffold-build-0 + - name: skaffold-build-1-task + resources: + inputs: + - name: source + resource: source-repo + outputs: + - name: source + resource: source-repo + taskRef: + name: skaffold-build-1 + - name: skaffold-deploy-0-task + resources: + inputs: + - from: + - skaffold-build-0-task + name: source + resource: source-repo + taskRef: + name: skaffold-deploy-0 + - name: skaffold-deploy-1-task + resources: + inputs: + - from: + - skaffold-build-1-task + name: source + resource: source-repo + taskRef: + name: skaffold-deploy-1 +status: {} +--- diff --git a/integration/testdata/generate_pipeline/multiple_configs/expectedSkaffold.yaml b/integration/testdata/generate_pipeline/multiple_configs/expectedSkaffold.yaml new file mode 100644 index 00000000000..519c69137f9 --- /dev/null +++ b/integration/testdata/generate_pipeline/multiple_configs/expectedSkaffold.yaml @@ -0,0 +1,22 @@ +apiVersion: skaffold/v1beta13 +kind: Config +build: + artifacts: + - image: gcr.io/k8s-skaffold/main-config +deploy: + kubectl: + manifests: + - k8s-* +profiles: +- name: oncluster + build: + artifacts: + - image: gcr.io/k8s-skaffold/main-config-pipeline + context: . + kaniko: + buildContext: + gcsBucket: skaffold-kaniko + tagPolicy: + gitCommit: {} + cluster: + pullSecretName: kaniko-secret \ No newline at end of file diff --git a/integration/testdata/generate_pipeline/multiple_configs/skaffold.yaml b/integration/testdata/generate_pipeline/multiple_configs/skaffold.yaml new file mode 100644 index 00000000000..519c69137f9 --- /dev/null +++ b/integration/testdata/generate_pipeline/multiple_configs/skaffold.yaml @@ -0,0 +1,22 @@ +apiVersion: skaffold/v1beta13 +kind: Config +build: + artifacts: + - image: gcr.io/k8s-skaffold/main-config +deploy: + kubectl: + manifests: + - k8s-* +profiles: +- name: oncluster + build: + artifacts: + - image: gcr.io/k8s-skaffold/main-config-pipeline + context: . + kaniko: + buildContext: + gcsBucket: skaffold-kaniko + tagPolicy: + gitCommit: {} + cluster: + pullSecretName: kaniko-secret \ No newline at end of file diff --git a/integration/testdata/generate_pipeline/multiple_configs/sub-app/skaffold.yaml b/integration/testdata/generate_pipeline/multiple_configs/sub-app/skaffold.yaml new file mode 100644 index 00000000000..39f42e9b484 --- /dev/null +++ b/integration/testdata/generate_pipeline/multiple_configs/sub-app/skaffold.yaml @@ -0,0 +1,9 @@ +apiVersion: skaffold/v1beta13 +kind: Config +build: + artifacts: + - image: gcr.io/k8s-skaffold/extra-config +deploy: + kubectl: + manifests: + - k8s-* \ No newline at end of file diff --git a/integration/testdata/generate_pipeline/no_profiles/expectedPipeline.yaml b/integration/testdata/generate_pipeline/no_profiles/expectedPipeline.yaml index 5455e0eaef3..f00b440d981 100644 --- a/integration/testdata/generate_pipeline/no_profiles/expectedPipeline.yaml +++ b/integration/testdata/generate_pipeline/no_profiles/expectedPipeline.yaml @@ -14,7 +14,7 @@ apiVersion: tekton.dev/v1alpha1 kind: Task metadata: creationTimestamp: null - name: skaffold-build + name: skaffold-build-0 spec: inputs: resources: @@ -58,7 +58,7 @@ apiVersion: tekton.dev/v1alpha1 kind: Task metadata: creationTimestamp: null - name: skaffold-deploy + name: skaffold-deploy-0 spec: inputs: resources: @@ -90,7 +90,7 @@ spec: - name: source-repo type: git tasks: - - name: skaffold-build-task + - name: skaffold-build-0-task resources: inputs: - name: source @@ -99,15 +99,15 @@ spec: - name: source resource: source-repo taskRef: - name: skaffold-build - - name: skaffold-deploy-task + name: skaffold-build-0 + - name: skaffold-deploy-0-task resources: inputs: - from: - - skaffold-build-task + - skaffold-build-0-task name: source resource: source-repo taskRef: - name: skaffold-deploy + name: skaffold-deploy-0 status: {} --- diff --git a/pkg/skaffold/generate_pipeline/generate_pipeline.go b/pkg/skaffold/generate_pipeline/generate_pipeline.go index 1edefcaeae7..ca1429a5153 100644 --- a/pkg/skaffold/generate_pipeline/generate_pipeline.go +++ b/pkg/skaffold/generate_pipeline/generate_pipeline.go @@ -28,12 +28,10 @@ import ( "github.com/ghodss/yaml" "github.com/pkg/errors" + tekton "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/pipeline" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest" - "github.com/GoogleContainerTools/skaffold/pkg/skaffold/version" - - tekton "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1" - corev1 "k8s.io/api/core/v1" ) // ConfigFile keeps track of config files and their corresponding SkaffoldConfigs and generated Profiles @@ -43,7 +41,7 @@ type ConfigFile struct { Profile *latest.Profile } -func Yaml(out io.Writer, configFile *ConfigFile) (*bytes.Buffer, error) { +func Yaml(out io.Writer, configFiles []*ConfigFile) (*bytes.Buffer, error) { // Generate git resource for pipeline gitResource, err := generateGitResource() if err != nil { @@ -52,18 +50,18 @@ func Yaml(out io.Writer, configFile *ConfigFile) (*bytes.Buffer, error) { // Generate build task for pipeline var tasks []*tekton.Task - taskBuild, err := generateBuildTask(configFile) + buildTasks, err := generateBuildTasks(configFiles) if err != nil { return nil, errors.Wrap(err, "generating build task") } - tasks = append(tasks, taskBuild) + tasks = append(tasks, buildTasks...) // Generate deploy task for pipeline - taskDeploy, err := generateDeployTask(configFile) + deployTasks, err := generateDeployTasks(configFiles) if err != nil { return nil, errors.Wrap(err, "generating deploy task") } - tasks = append(tasks, taskDeploy) + tasks = append(tasks, deployTasks...) // Generate pipeline from git resource and tasks pipeline, err := generatePipeline(tasks) @@ -117,103 +115,6 @@ func generateGitResource() (*tekton.PipelineResource, error) { return pipeline.NewGitResource("source-git", gitURL), nil } -func generateBuildTask(configFile *ConfigFile) (*tekton.Task, error) { - buildConfig := configFile.Profile.Build - if len(buildConfig.Artifacts) == 0 { - return nil, errors.New("no artifacts to build") - } - - skaffoldVersion := os.Getenv("PIPELINE_SKAFFOLD_VERSION") - if skaffoldVersion == "" { - skaffoldVersion = version.Get().Version - } - - resources := []tekton.TaskResource{ - { - Name: "source", - Type: tekton.PipelineResourceTypeGit, - }, - } - inputs := &tekton.Inputs{Resources: resources} - outputs := &tekton.Outputs{Resources: resources} - steps := []corev1.Container{ - { - Name: "run-build", - Image: fmt.Sprintf("gcr.io/k8s-skaffold/skaffold:%s", skaffoldVersion), - WorkingDir: "/workspace/source", - Command: []string{"skaffold", "build"}, - Args: []string{ - "--filename", configFile.Path, - "--profile", "oncluster", - "--file-output", "build.out", - }, - }, - } - - // Add secret volume mounting for artifacts that need to be built with kaniko - var volumes []corev1.Volume - if buildConfig.Artifacts[0].KanikoArtifact != nil { - volumes = []corev1.Volume{ - { - Name: kanikoSecretName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: kanikoSecretName, - }, - }, - }, - } - steps[0].VolumeMounts = []corev1.VolumeMount{ - { - Name: kanikoSecretName, - MountPath: "/secret", - }, - } - steps[0].Env = []corev1.EnvVar{ - { - Name: "GOOGLE_APPLICATION_CREDENTIALS", - Value: "/secret/" + kanikoSecretName, - }, - } - } - - return pipeline.NewTask("skaffold-build", inputs, outputs, steps, volumes), nil -} - -func generateDeployTask(configFile *ConfigFile) (*tekton.Task, error) { - deployConfig := configFile.Config.Deploy - if deployConfig.HelmDeploy == nil && deployConfig.KubectlDeploy == nil && deployConfig.KustomizeDeploy == nil { - return nil, errors.New("no Helm/Kubectl/Kustomize deploy config") - } - - skaffoldVersion := os.Getenv("PIPELINE_SKAFFOLD_VERSION") - if skaffoldVersion == "" { - skaffoldVersion = version.Get().Version - } - - resources := []tekton.TaskResource{ - { - Name: "source", - Type: tekton.PipelineResourceTypeGit, - }, - } - inputs := &tekton.Inputs{Resources: resources} - steps := []corev1.Container{ - { - Name: "run-deploy", - Image: fmt.Sprintf("gcr.io/k8s-skaffold/skaffold:%s", skaffoldVersion), - WorkingDir: "/workspace/source", - Command: []string{"skaffold", "deploy"}, - Args: []string{ - "--filename", configFile.Path, - "--build-artifacts", "build.out", - }, - }, - } - - return pipeline.NewTask("skaffold-deploy", inputs, nil, steps, nil), nil -} - func generatePipeline(tasks []*tekton.Task) (*tekton.Pipeline, error) { if len(tasks) == 0 { return nil, errors.New("no tasks to add to pipeline") @@ -227,7 +128,7 @@ func generatePipeline(tasks []*tekton.Task) (*tekton.Pipeline, error) { } // Create tasks in pipeline spec for all corresponding tasks pipelineTasks := make([]tekton.PipelineTask, 0) - for i, task := range tasks { + for _, task := range tasks { pipelineTask := tekton.PipelineTask{ Name: fmt.Sprintf("%s-task", task.Name), TaskRef: tekton.TaskRef{ @@ -251,7 +152,9 @@ func generatePipeline(tasks []*tekton.Task) (*tekton.Pipeline, error) { }, } } else { - pipelineTask.Resources.Inputs[0].From = []string{pipelineTasks[i-1].Name} + // Get the git resource for deploy commands from their corresponding build command + from := strings.Replace(pipelineTask.Name, "deploy", "build", 1) + pipelineTask.Resources.Inputs[0].From = []string{from} } pipelineTasks = append(pipelineTasks, pipelineTask) diff --git a/pkg/skaffold/generate_pipeline/generate_pipeline_test.go b/pkg/skaffold/generate_pipeline/generate_pipeline_test.go index bf7cd410f87..ad5f10dd834 100644 --- a/pkg/skaffold/generate_pipeline/generate_pipeline_test.go +++ b/pkg/skaffold/generate_pipeline/generate_pipeline_test.go @@ -19,7 +19,6 @@ package generatepipeline import ( "testing" - "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest" "github.com/GoogleContainerTools/skaffold/testutil" tekton "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1" @@ -166,90 +165,3 @@ func TestGeneratePipeline(t *testing.T) { }) } } - -func TestGenerateBuildTask(t *testing.T) { - var tests = []struct { - description string - buildConfig latest.BuildConfig - shouldErr bool - }{ - { - description: "successfully generate build task", - buildConfig: latest.BuildConfig{ - Artifacts: []*latest.Artifact{ - { - ImageName: "testArtifact", - }, - }, - }, - shouldErr: false, - }, - { - description: "fail generating build task", - buildConfig: latest.BuildConfig{ - Artifacts: []*latest.Artifact{}, - }, - shouldErr: true, - }, - } - - for _, test := range tests { - testutil.Run(t, test.description, func(t *testutil.T) { - configFile := &ConfigFile{ - Path: "test", - Profile: &latest.Profile{ - Pipeline: latest.Pipeline{ - Build: test.buildConfig, - }, - }, - } - _, err := generateBuildTask(configFile) - t.CheckError(test.shouldErr, err) - }) - } -} - -func TestGenerateDeployTask(t *testing.T) { - var tests = []struct { - description string - deployConfig latest.DeployConfig - shouldErr bool - }{ - { - description: "successfully generate deploy task", - deployConfig: latest.DeployConfig{ - DeployType: latest.DeployType{ - HelmDeploy: &latest.HelmDeploy{}, - }, - }, - shouldErr: false, - }, - { - description: "fail generating deploy task", - deployConfig: latest.DeployConfig{ - DeployType: latest.DeployType{ - HelmDeploy: nil, - KubectlDeploy: nil, - KustomizeDeploy: nil, - }, - }, - shouldErr: true, - }, - } - - for _, test := range tests { - testutil.Run(t, test.description, func(t *testutil.T) { - configFile := &ConfigFile{ - Path: "test", - Config: &latest.SkaffoldConfig{ - Pipeline: latest.Pipeline{ - Deploy: test.deployConfig, - }, - }, - } - - _, err := generateDeployTask(configFile) - t.CheckError(test.shouldErr, err) - }) - } -} diff --git a/pkg/skaffold/generate_pipeline/tasks.go b/pkg/skaffold/generate_pipeline/tasks.go new file mode 100644 index 00000000000..135a0b5ecc0 --- /dev/null +++ b/pkg/skaffold/generate_pipeline/tasks.go @@ -0,0 +1,159 @@ +/* +Copyright 2019 The Skaffold Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generatepipeline + +import ( + "fmt" + "os" + + "github.com/pkg/errors" + + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/pipeline" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/version" + + tekton "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1" + corev1 "k8s.io/api/core/v1" +) + +func generateBuildTasks(configFiles []*ConfigFile) ([]*tekton.Task, error) { + var tasks []*tekton.Task + for i, configFile := range configFiles { + task, err := generateBuildTask(configFile) + if err != nil { + return nil, err + } + task.Name = fmt.Sprintf("%s-%d", task.Name, i) + + tasks = append(tasks, task) + } + + return tasks, nil +} + +func generateBuildTask(configFile *ConfigFile) (*tekton.Task, error) { + buildConfig := configFile.Profile.Build + if len(buildConfig.Artifacts) == 0 { + return nil, errors.New("no artifacts to build") + } + + skaffoldVersion := os.Getenv("PIPELINE_SKAFFOLD_VERSION") + if skaffoldVersion == "" { + skaffoldVersion = version.Get().Version + } + + resources := []tekton.TaskResource{ + { + Name: "source", + Type: tekton.PipelineResourceTypeGit, + }, + } + inputs := &tekton.Inputs{Resources: resources} + outputs := &tekton.Outputs{Resources: resources} + steps := []corev1.Container{ + { + Name: "run-build", + Image: fmt.Sprintf("gcr.io/k8s-skaffold/skaffold:%s", skaffoldVersion), + WorkingDir: "/workspace/source", + Command: []string{"skaffold", "build"}, + Args: []string{ + "--filename", configFile.Path, + "--profile", "oncluster", + "--file-output", "build.out", + }, + }, + } + + // Add secret volume mounting if any artifacts in config need to be built with kaniko + var volumes []corev1.Volume + for _, artifact := range buildConfig.Artifacts { + if artifact.KanikoArtifact != nil { + volumes = []corev1.Volume{ + { + Name: kanikoSecretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: kanikoSecretName, + }, + }, + }, + } + steps[0].VolumeMounts = []corev1.VolumeMount{ + { + Name: kanikoSecretName, + MountPath: "/secret", + }, + } + steps[0].Env = []corev1.EnvVar{ + { + Name: "GOOGLE_APPLICATION_CREDENTIALS", + Value: "/secret/" + kanikoSecretName, + }, + } + } + } + + return pipeline.NewTask("skaffold-build", inputs, outputs, steps, volumes), nil +} + +func generateDeployTasks(configFiles []*ConfigFile) ([]*tekton.Task, error) { + var tasks []*tekton.Task + for i, configFile := range configFiles { + task, err := generateDeployTask(configFile) + if err != nil { + return nil, err + } + task.Name = fmt.Sprintf("%s-%d", task.Name, i) + + tasks = append(tasks, task) + } + + return tasks, nil +} + +func generateDeployTask(configFile *ConfigFile) (*tekton.Task, error) { + deployConfig := configFile.Config.Deploy + if deployConfig.HelmDeploy == nil && deployConfig.KubectlDeploy == nil && deployConfig.KustomizeDeploy == nil { + return nil, errors.New("no Helm/Kubectl/Kustomize deploy config") + } + + skaffoldVersion := os.Getenv("PIPELINE_SKAFFOLD_VERSION") + if skaffoldVersion == "" { + skaffoldVersion = version.Get().Version + } + + resources := []tekton.TaskResource{ + { + Name: "source", + Type: tekton.PipelineResourceTypeGit, + }, + } + inputs := &tekton.Inputs{Resources: resources} + steps := []corev1.Container{ + { + Name: "run-deploy", + Image: fmt.Sprintf("gcr.io/k8s-skaffold/skaffold:%s", skaffoldVersion), + WorkingDir: "/workspace/source", + Command: []string{"skaffold", "deploy"}, + Args: []string{ + "--filename", configFile.Path, + "--build-artifacts", "build.out", + }, + }, + } + + return pipeline.NewTask("skaffold-deploy", inputs, nil, steps, nil), nil +} diff --git a/pkg/skaffold/generate_pipeline/tasks_test.go b/pkg/skaffold/generate_pipeline/tasks_test.go new file mode 100644 index 00000000000..38e58ec0d89 --- /dev/null +++ b/pkg/skaffold/generate_pipeline/tasks_test.go @@ -0,0 +1,207 @@ +/* +Copyright 2019 The Skaffold Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package generatepipeline + +import ( + "testing" + + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest" + "github.com/GoogleContainerTools/skaffold/testutil" +) + +func TestGenerateBuildTasks(t *testing.T) { + var tests = []struct { + description string + configFiles []*ConfigFile + shouldErr bool + }{ + { + description: "successfully generate build tasks", + configFiles: []*ConfigFile{ + { + Path: "test1", + Profile: &latest.Profile{ + Pipeline: latest.Pipeline{ + Build: latest.BuildConfig{ + Artifacts: []*latest.Artifact{ + { + ImageName: "testArtifact1", + }, + }, + }, + }, + }, + }, + { + Path: "test2", + Profile: &latest.Profile{ + Pipeline: latest.Pipeline{ + Build: latest.BuildConfig{ + Artifacts: []*latest.Artifact{ + { + ImageName: "testArtifact2", + }, + }, + }, + }, + }, + }, + }, + shouldErr: false, + }, + } + + for _, test := range tests { + testutil.Run(t, test.description, func(t *testutil.T) { + _, err := generateBuildTasks(test.configFiles) + t.CheckError(test.shouldErr, err) + }) + } +} + +func TestGenerateBuildTask(t *testing.T) { + var tests = []struct { + description string + buildConfig latest.BuildConfig + shouldErr bool + }{ + { + description: "successfully generate build task", + buildConfig: latest.BuildConfig{ + Artifacts: []*latest.Artifact{ + { + ImageName: "testArtifact", + }, + }, + }, + shouldErr: false, + }, + { + description: "fail generating build task", + buildConfig: latest.BuildConfig{ + Artifacts: []*latest.Artifact{}, + }, + shouldErr: true, + }, + } + + for _, test := range tests { + testutil.Run(t, test.description, func(t *testutil.T) { + configFile := &ConfigFile{ + Path: "test", + Profile: &latest.Profile{ + Pipeline: latest.Pipeline{ + Build: test.buildConfig, + }, + }, + } + _, err := generateBuildTask(configFile) + t.CheckError(test.shouldErr, err) + }) + } +} + +func TestGenerateDeployTasks(t *testing.T) { + var tests = []struct { + description string + configFiles []*ConfigFile + shouldErr bool + }{ + { + description: "successfully generate deploy tasks", + configFiles: []*ConfigFile{ + { + Path: "test1", + Config: &latest.SkaffoldConfig{ + Pipeline: latest.Pipeline{ + Deploy: latest.DeployConfig{ + DeployType: latest.DeployType{ + HelmDeploy: &latest.HelmDeploy{}, + }, + }, + }, + }, + }, + { + Path: "test2", + Config: &latest.SkaffoldConfig{ + Pipeline: latest.Pipeline{ + Deploy: latest.DeployConfig{ + DeployType: latest.DeployType{ + HelmDeploy: &latest.HelmDeploy{}, + }, + }, + }, + }, + }, + }, + shouldErr: false, + }, + } + + for _, test := range tests { + testutil.Run(t, test.description, func(t *testutil.T) { + _, err := generateDeployTasks(test.configFiles) + t.CheckError(test.shouldErr, err) + }) + } +} + +func TestGenerateDeployTask(t *testing.T) { + var tests = []struct { + description string + deployConfig latest.DeployConfig + shouldErr bool + }{ + { + description: "successfully generate deploy task", + deployConfig: latest.DeployConfig{ + DeployType: latest.DeployType{ + HelmDeploy: &latest.HelmDeploy{}, + }, + }, + shouldErr: false, + }, + { + description: "fail generating deploy task", + deployConfig: latest.DeployConfig{ + DeployType: latest.DeployType{ + HelmDeploy: nil, + KubectlDeploy: nil, + KustomizeDeploy: nil, + }, + }, + shouldErr: true, + }, + } + + for _, test := range tests { + testutil.Run(t, test.description, func(t *testutil.T) { + configFile := &ConfigFile{ + Path: "test", + Config: &latest.SkaffoldConfig{ + Pipeline: latest.Pipeline{ + Deploy: test.deployConfig, + }, + }, + } + + _, err := generateDeployTask(configFile) + t.CheckError(test.shouldErr, err) + }) + } +} diff --git a/pkg/skaffold/runner/generate_pipeline.go b/pkg/skaffold/runner/generate_pipeline.go index 4111594a56b..eeb259758a5 100644 --- a/pkg/skaffold/runner/generate_pipeline.go +++ b/pkg/skaffold/runner/generate_pipeline.go @@ -22,6 +22,8 @@ import ( "io/ioutil" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/color" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema" + "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/defaults" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest" "github.com/pkg/errors" @@ -29,22 +31,32 @@ import ( pipeline "github.com/GoogleContainerTools/skaffold/pkg/skaffold/generate_pipeline" ) -func (r *SkaffoldRunner) GeneratePipeline(ctx context.Context, out io.Writer, config *latest.SkaffoldConfig, fileOut string) error { +func (r *SkaffoldRunner) GeneratePipeline(ctx context.Context, out io.Writer, config *latest.SkaffoldConfig, configPaths []string, fileOut string) error { // Keep track of files, configs, and profiles. This will be used to know which files to write // profiles to and what flags to add to task commands - configFile := &pipeline.ConfigFile{ - Path: r.runCtx.Opts.ConfigurationFile, - Config: config, - Profile: nil, + baseConfig := []*pipeline.ConfigFile{ + { + Path: r.runCtx.Opts.ConfigurationFile, + Config: config, + Profile: nil, + }, } + configFiles, err := setupConfigFiles(configPaths) + if err != nil { + return errors.Wrap(err, "setting up ConfigFiles") + } + configFiles = append(baseConfig, configFiles...) + // Will run the profile setup multiple times and require user input for each specified config color.Default.Fprintln(out, "Running profile setup...") - if err := pipeline.CreateSkaffoldProfile(out, configFile); err != nil { - return errors.Wrap(err, "seeting up profile") + for _, configFile := range configFiles { + if err := pipeline.CreateSkaffoldProfile(out, configFile); err != nil { + return errors.Wrap(err, "seeting up profile") + } } color.Default.Fprintln(out, "Generating Pipeline...") - pipelineYaml, err := pipeline.Yaml(out, configFile) + pipelineYaml, err := pipeline.Yaml(out, configFiles) if err != nil { return errors.Wrap(err, "generating pipeline yaml contents") } @@ -52,3 +64,31 @@ func (r *SkaffoldRunner) GeneratePipeline(ctx context.Context, out io.Writer, co // write all yaml pieces to output return ioutil.WriteFile(fileOut, pipelineYaml.Bytes(), 0755) } + +func setupConfigFiles(configPaths []string) ([]*pipeline.ConfigFile, error) { + if configPaths == nil { + return []*pipeline.ConfigFile{}, nil + } + + // Read all given config files to read contents into SkaffoldConfig + var configFiles []*pipeline.ConfigFile + for _, path := range configPaths { + parsed, err := schema.ParseConfig(path, true) + if err != nil { + return nil, errors.Wrapf(err, "parsing config %s", path) + } + config := parsed.(*latest.SkaffoldConfig) + + if err := defaults.Set(config); err != nil { + return nil, errors.Wrap(err, "setting default values for extra configs") + } + + configFile := &pipeline.ConfigFile{ + Path: path, + Config: config, + } + configFiles = append(configFiles, configFile) + } + + return configFiles, nil +} diff --git a/pkg/skaffold/runner/runner.go b/pkg/skaffold/runner/runner.go index 890d2e236f6..f62111761e9 100644 --- a/pkg/skaffold/runner/runner.go +++ b/pkg/skaffold/runner/runner.go @@ -39,7 +39,7 @@ type Runner interface { Dev(context.Context, io.Writer, []*latest.Artifact) error BuildAndTest(context.Context, io.Writer, []*latest.Artifact) ([]build.Artifact, error) DeployAndLog(context.Context, io.Writer, []build.Artifact) error - GeneratePipeline(context.Context, io.Writer, *latest.SkaffoldConfig, string) error + GeneratePipeline(context.Context, io.Writer, *latest.SkaffoldConfig, []string, string) error Cleanup(context.Context, io.Writer) error Prune(context.Context, io.Writer) error HasDeployed() bool