diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c5ca376d2..fc32ee1502 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,5 @@ name: Unit, Integration, and E2E Tests -on: +on: pull_request: branches: - main @@ -7,7 +7,7 @@ on: paths-ignore: - 'README.md' - 'docs/**' - branches: + branches: - main jobs: @@ -114,8 +114,9 @@ jobs: # host.docker.internal does not work in a GitHub action docker exec kind-control-plane bash -c "echo '172.17.0.1 host.docker.internal' >>/etc/hosts" - # Build and load the Git image + # Build and load the Git and Bundle image export GIT_CONTAINER_IMAGE="$(KO_DOCKER_REPO=kind.local ko publish ./cmd/git)" + export BUNDLE_CONTAINER_IMAGE="$(KO_DOCKER_REPO=kind.local ko publish ./cmd/bundle)" make test-integration diff --git a/cmd/bundle/main.go b/cmd/bundle/main.go index 21a5c09399..1db85beafc 100644 --- a/cmd/bundle/main.go +++ b/cmd/bundle/main.go @@ -9,8 +9,11 @@ import ( "fmt" "log" "os" + "strconv" "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/spf13/pflag" "github.com/shipwright-io/build/pkg/bundle" @@ -18,12 +21,13 @@ import ( ) type settings struct { - help bool - image string - prune bool - target string - secretPath string - resultFileImageDigest string + help bool + image string + prune bool + target string + secretPath string + resultFileImageDigest string + resultFileSourceTimestamp string } var flagValues settings @@ -36,6 +40,7 @@ func init() { pflag.StringVar(&flagValues.image, "image", "", "Location of the bundle image (mandatory)") pflag.StringVar(&flagValues.target, "target", "/workspace/source", "The target directory to place the code") pflag.StringVar(&flagValues.resultFileImageDigest, "result-file-image-digest", "", "A file to write the image digest") + pflag.StringVar(&flagValues.resultFileSourceTimestamp, "result-file-source-timestamp", "", "A file to write the source timestamp") pflag.StringVar(&flagValues.secretPath, "secret-path", "", "A directory that contains access credentials (optional)") pflag.BoolVar(&flagValues.prune, "prune", false, "Delete bundle image from registry after it was pulled") @@ -72,10 +77,20 @@ func Do(ctx context.Context) error { } log.Printf("Pulling image %q", ref) - img, err := bundle.PullAndUnpack( - ref, - flagValues.target, - options...) + desc, err := remote.Get(ref, options...) + if err != nil { + return err + } + + img, err := desc.Image() + if err != nil { + return err + } + + rc := mutate.Extract(img) + defer rc.Close() + + unpackDetails, err := bundle.Unpack(rc, flagValues.target) if err != nil { return err } @@ -93,6 +108,17 @@ func Do(ctx context.Context) error { } } + if flagValues.resultFileSourceTimestamp != "" { + if unpackDetails.MostRecentFileTimestamp != nil { + if err = os.WriteFile(flagValues.resultFileSourceTimestamp, []byte(strconv.FormatInt(unpackDetails.MostRecentFileTimestamp.Unix(), 10)), 0644); err != nil { + return err + } + + } else { + log.Printf("Unable to determine source timestamp of content in %s\n", flagValues.target) + } + } + if flagValues.prune { // Some container registry implementations, i.e. library/registry:2 will fail to // delete the image when there is no image digest given. Use image digest from the diff --git a/cmd/bundle/main_test.go b/cmd/bundle/main_test.go index 49213a9006..d1cc01ae83 100644 --- a/cmd/bundle/main_test.go +++ b/cmd/bundle/main_test.go @@ -9,25 +9,31 @@ import ( "fmt" "io" "log" + "net/http/httptest" + "net/url" "os" "path/filepath" + "time" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "github.com/shipwright-io/build/cmd/bundle" - "github.com/shipwright-io/build/pkg/image" "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" containerreg "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" "k8s.io/apimachinery/pkg/util/rand" + + "github.com/shipwright-io/build/pkg/bundle" + "github.com/shipwright-io/build/pkg/image" ) var _ = Describe("Bundle Loader", func() { const exampleImage = "ghcr.io/shipwright-io/sample-go/source-bundle:latest" - var run = func(args ...string) error { + run := func(args ...string) error { // discard log output log.SetOutput(io.Discard) @@ -40,7 +46,7 @@ var _ = Describe("Bundle Loader", func() { return Do(context.Background()) } - var withTempDir = func(f func(target string)) { + withTempDir := func(f func(target string)) { path, err := os.MkdirTemp(os.TempDir(), "bundle") Expect(err).ToNot(HaveOccurred()) defer os.RemoveAll(path) @@ -56,6 +62,24 @@ var _ = Describe("Bundle Loader", func() { f(file.Name()) } + withTempRegistry := func(f func(endpoint string)) { + logLogger := log.Logger{} + logLogger.SetOutput(GinkgoWriter) + + s := httptest.NewServer( + registry.New( + registry.Logger(&logLogger), + registry.WithReferrersSupport(true), + ), + ) + defer s.Close() + + u, err := url.Parse(s.URL) + Expect(err).ToNot(HaveOccurred()) + + f(u.Host) + } + filecontent := func(path string) string { data, err := os.ReadFile(path) Expect(err).ToNot(HaveOccurred()) @@ -188,14 +212,16 @@ var _ = Describe("Bundle Loader", func() { }) AfterEach(func() { - ref, err := name.ParseReference(testImage) - Expect(err).ToNot(HaveOccurred()) + if testImage != "" { + ref, err := name.ParseReference(testImage) + Expect(err).ToNot(HaveOccurred()) - options, auth, err := image.GetOptions(context.TODO(), ref, true, dockerConfigFile, "test-agent") - Expect(err).ToNot(HaveOccurred()) + options, auth, err := image.GetOptions(context.TODO(), ref, true, dockerConfigFile, "test-agent") + Expect(err).ToNot(HaveOccurred()) - // Delete test image (best effort) - _ = image.Delete(ref, options, *auth) + // Delete test image (best effort) + _ = image.Delete(ref, options, *auth) + } }) It("should pull and unpack an image from a private registry", func() { @@ -232,4 +258,62 @@ var _ = Describe("Bundle Loader", func() { }) }) }) + + Context("Result file checks", func() { + tmpFile := func(dir string, name string, data []byte, timestamp time.Time) { + var path = filepath.Join(dir, name) + + Expect(os.WriteFile( + path, + data, + os.FileMode(0644), + )).To(Succeed()) + + Expect(os.Chtimes( + path, + timestamp, + timestamp, + )).To(Succeed()) + } + + // Creates a controlled reference image with one file called "file" with modification + // timestamp of Friday, February 13, 2009 11:31:30 PM (unix timestamp 1234567890) + withReferenceImage := func(f func(dig name.Digest)) { + withTempRegistry(func(endpoint string) { + withTempDir(func(target string) { + timestamp := time.Unix(1234567890, 0) + + ref, err := name.ParseReference(fmt.Sprintf("%s/namespace/image:tag", endpoint)) + Expect(err).ToNot(HaveOccurred()) + Expect(ref).ToNot(BeNil()) + + tmpFile(target, "file", []byte("foobar"), timestamp) + + dig, err := bundle.PackAndPush(ref, target) + Expect(err).ToNot(HaveOccurred()) + Expect(dig).ToNot(BeNil()) + + f(dig) + }) + }) + } + + It("should store source timestamp in result file", func() { + withTempDir(func(target string) { + withTempDir(func(result string) { + withReferenceImage(func(dig name.Digest) { + resultSourceTimestamp := filepath.Join(result, "source-timestamp") + + Expect(run( + "--image", dig.String(), + "--target", target, + "--result-file-source-timestamp", resultSourceTimestamp, + )).To(Succeed()) + + Expect(filecontent(resultSourceTimestamp)).To(Equal("1234567890")) + }) + }) + }) + }) + }) }) diff --git a/cmd/git/main.go b/cmd/git/main.go index ff12f9107b..c1d0e6851b 100644 --- a/cmd/git/main.go +++ b/cmd/git/main.go @@ -46,20 +46,21 @@ func (e ExitError) Error() string { } type settings struct { - help bool - url string - revision string - depth uint - target string - resultFileCommitSha string - resultFileCommitAuthor string - resultFileBranchName string - secretPath string - skipValidation bool - gitURLRewrite bool - resultFileErrorMessage string - resultFileErrorReason string - verbose bool + help bool + url string + revision string + depth uint + target string + resultFileCommitSha string + resultFileCommitAuthor string + resultFileBranchName string + resultFileSourceTimestamp string + secretPath string + skipValidation bool + gitURLRewrite bool + resultFileErrorMessage string + resultFileErrorReason string + verbose bool } var flagValues settings @@ -81,6 +82,7 @@ func init() { pflag.StringVar(&flagValues.target, "target", "", "The target directory of the clone operation") pflag.StringVar(&flagValues.resultFileCommitSha, "result-file-commit-sha", "", "A file to write the commit sha to.") pflag.StringVar(&flagValues.resultFileCommitAuthor, "result-file-commit-author", "", "A file to write the commit author to.") + pflag.StringVar(&flagValues.resultFileSourceTimestamp, "result-file-source-timestamp", "", "A file to write the source timestamp to.") pflag.StringVar(&flagValues.resultFileBranchName, "result-file-branch-name", "", "A file to write the branch name to.") pflag.StringVar(&flagValues.secretPath, "secret-path", "", "A directory that contains a secret. Either username and password for basic authentication. Or a SSH private key and optionally a known hosts file. Optional.") @@ -180,6 +182,17 @@ func runGitClone(ctx context.Context) error { } } + if flagValues.resultFileSourceTimestamp != "" { + output, err := git(ctx, "-C", flagValues.target, "show", "--no-patch", "--format=%ct") + if err != nil { + return err + } + + if err = os.WriteFile(flagValues.resultFileSourceTimestamp, []byte(output), 0644); err != nil { + return err + } + } + if strings.TrimSpace(flagValues.revision) == "" && strings.TrimSpace(flagValues.resultFileBranchName) != "" { output, err := git(ctx, "-C", flagValues.target, "rev-parse", "--abbrev-ref", "HEAD") if err != nil { diff --git a/cmd/git/main_test.go b/cmd/git/main_test.go index 5c8de99fb0..d50543a05c 100644 --- a/cmd/git/main_test.go +++ b/cmd/git/main_test.go @@ -471,6 +471,21 @@ var _ = Describe("Git Resource", func() { }) }) }) + + It("should store source-timestamp into file specified in --result-file-source-timestamp flag", func() { + withTempFile("source-timestamp", func(filename string) { + withTempDir(func(target string) { + Expect(run(withArgs( + "--url", exampleRepo, + "--target", target, + "--revision", "v0.1.0", + "--result-file-source-timestamp", filename, + ))).ToNot(HaveOccurred()) + + Expect(filecontent(filename)).To(Equal("1619426578")) + }) + }) + }) }) Context("Some tests mutate or depend on git configurations. They must run sequentially to avoid race-conditions.", Ordered, func() { diff --git a/deploy/crds/shipwright.io_buildruns.yaml b/deploy/crds/shipwright.io_buildruns.yaml index d3cb6e5b2e..e0b710e8e8 100644 --- a/deploy/crds/shipwright.io_buildruns.yaml +++ b/deploy/crds/shipwright.io_buildruns.yaml @@ -6334,6 +6334,13 @@ spec: name: description: Name is the name of source type: string + timestamp: + description: Timestamp holds the timestamp of the source, which + depends on the actual source type and could range from being + the commit timestamp or the fileystem timestamp of the most + recent source file in the working directory + format: date-time + type: string required: - name type: object @@ -12552,6 +12559,13 @@ spec: description: Digest hold the image digest result type: string type: object + timestamp: + description: Timestamp holds the timestamp of the source, which + depends on the actual source type and could range from being + the commit timestamp or the fileystem timestamp of the most + recent source file in the working directory + format: date-time + type: string type: object startTime: description: StartTime is the time the build is actually started. diff --git a/pkg/apis/build/v1alpha1/buildrun_types.go b/pkg/apis/build/v1alpha1/buildrun_types.go index f8458f82b2..2d8cc09a24 100644 --- a/pkg/apis/build/v1alpha1/buildrun_types.go +++ b/pkg/apis/build/v1alpha1/buildrun_types.go @@ -117,6 +117,14 @@ type SourceResult struct { // // +optional Bundle *BundleSourceResult `json:"bundle,omitempty"` + + // Timestamp holds the timestamp of the source, which + // depends on the actual source type and could range from + // being the commit timestamp or the fileystem timestamp + // of the most recent source file in the working directory + // + // +optional + Timestamp *metav1.Time `json:"timestamp,omitempty"` } // BundleSourceResult holds the results emitted from the bundle source diff --git a/pkg/apis/build/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/build/v1alpha1/zz_generated.deepcopy.go index a546cc9ef0..c36175df27 100644 --- a/pkg/apis/build/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/build/v1alpha1/zz_generated.deepcopy.go @@ -1123,6 +1123,10 @@ func (in *SourceResult) DeepCopyInto(out *SourceResult) { *out = new(BundleSourceResult) **out = **in } + if in.Timestamp != nil { + in, out := &in.Timestamp, &out.Timestamp + *out = (*in).DeepCopy() + } return } diff --git a/pkg/apis/build/v1beta1/buildrun_conversion.go b/pkg/apis/build/v1beta1/buildrun_conversion.go index 41180e9ae5..b7b3cac157 100644 --- a/pkg/apis/build/v1beta1/buildrun_conversion.go +++ b/pkg/apis/build/v1beta1/buildrun_conversion.go @@ -142,6 +142,7 @@ func (src *BuildRun) ConvertFrom(ctx context.Context, obj *unstructured.Unstruct sourceStatus = &SourceResult{ Git: (*GitSourceResult)(s.Git), OciArtifact: (*OciArtifactSourceResult)(s.Bundle), + Timestamp: s.Timestamp, } } diff --git a/pkg/apis/build/v1beta1/buildrun_types.go b/pkg/apis/build/v1beta1/buildrun_types.go index 7a40e822b0..8703976ff1 100644 --- a/pkg/apis/build/v1beta1/buildrun_types.go +++ b/pkg/apis/build/v1beta1/buildrun_types.go @@ -122,6 +122,14 @@ type SourceResult struct { // // +optional OciArtifact *OciArtifactSourceResult `json:"ociArtifact,omitempty"` + + // Timestamp holds the timestamp of the source, which + // depends on the actual source type and could range from + // being the commit timestamp or the fileystem timestamp + // of the most recent source file in the working directory + // + // +optional + Timestamp *metav1.Time `json:"timestamp,omitempty"` } // OciArtifactSourceResult holds the results emitted from the bundle source diff --git a/pkg/apis/build/v1beta1/zz_generated.deepcopy.go b/pkg/apis/build/v1beta1/zz_generated.deepcopy.go index 4000f8342c..e2fa2ff9c2 100644 --- a/pkg/apis/build/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/build/v1beta1/zz_generated.deepcopy.go @@ -1097,6 +1097,10 @@ func (in *SourceResult) DeepCopyInto(out *SourceResult) { *out = new(OciArtifactSourceResult) **out = **in } + if in.Timestamp != nil { + in, out := &in.Timestamp, &out.Timestamp + *out = (*in).DeepCopy() + } return } diff --git a/pkg/bundle/bundle.go b/pkg/bundle/bundle.go index 2a32ea78ba..384ba37225 100644 --- a/pkg/bundle/bundle.go +++ b/pkg/bundle/bundle.go @@ -26,6 +26,11 @@ import ( const shpIgnoreFilename = ".shpignore" +// UnpackDetails contains details about the files that were unpacked +type UnpackDetails struct { + MostRecentFileTimestamp *time.Time +} + // PackAndPush a local directory as-is into a container image. See // remote.Option for optional options to the image push to the registry, for // example to provide the appropriate access credentials. @@ -35,12 +40,12 @@ func PackAndPush(ref name.Reference, directory string, options ...remote.Option) return name.Digest{}, err } - image, err := mutate.AppendLayers(empty.Image, bundleLayer) + image, err := mutate.Time(empty.Image, time.Unix(0, 0)) if err != nil { return name.Digest{}, err } - image, err = mutate.Time(image, time.Unix(0, 0)) + image, err = mutate.AppendLayers(image, bundleLayer) if err != nil { return name.Digest{}, err } @@ -77,7 +82,7 @@ func PullAndUnpack(ref name.Reference, targetPath string, options ...remote.Opti rc := mutate.Extract(image) defer rc.Close() - if err = Unpack(rc, targetPath); err != nil { + if _, err = Unpack(rc, targetPath); err != nil { return nil, err } @@ -224,16 +229,17 @@ func Pack(directory string) (io.ReadCloser, error) { // Unpack reads a tar stream and writes the content into the local file system // with all files and directories. -func Unpack(in io.Reader, targetPath string) error { +func Unpack(in io.Reader, targetPath string) (*UnpackDetails, error) { + var details = UnpackDetails{} var tr = tar.NewReader(in) for { header, err := tr.Next() switch { case err == io.EOF: - return nil + return &details, nil case err != nil: - return err + return nil, err case header == nil: continue @@ -241,37 +247,46 @@ func Unpack(in io.Reader, targetPath string) error { var target = filepath.Join(targetPath, header.Name) if strings.Contains(target, "/../") { - return fmt.Errorf("targetPath validation failed, path contains unexpected special elements") + return nil, fmt.Errorf("targetPath validation failed, path contains unexpected special elements") } switch header.Typeflag { case tar.TypeDir: if err := os.MkdirAll(target, os.FileMode(header.Mode)); err != nil { - return err + return nil, err } case tar.TypeReg: // Edge case in which that tarball did not have a directory entry dir, _ := filepath.Split(target) if err := os.MkdirAll(dir, os.FileMode(0755)); err != nil { - return err + return nil, err } file, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode)) if err != nil { - return err + return nil, err } if _, err := io.Copy(file, tr); err != nil { file.Close() - return err + return nil, err + } + + if err := file.Close(); err != nil { + return nil, err } - file.Close() - os.Chtimes(target, header.AccessTime, header.ModTime) + if err := os.Chtimes(target, header.AccessTime, header.ModTime); err != nil { + return nil, err + } + + if details.MostRecentFileTimestamp == nil || details.MostRecentFileTimestamp.Before(header.ModTime) { + details.MostRecentFileTimestamp = &header.ModTime + } default: - return fmt.Errorf("provided tarball contains unsupported file type, only directories and regular files are supported") + return nil, fmt.Errorf("provided tarball contains unsupported file type, only directories and regular files are supported") } } } diff --git a/pkg/bundle/bundle_test.go b/pkg/bundle/bundle_test.go index 870a3a9d4e..cd980ae5cb 100644 --- a/pkg/bundle/bundle_test.go +++ b/pkg/bundle/bundle_test.go @@ -54,7 +54,10 @@ var _ = Describe("Bundle", func() { Expect(err).ToNot(HaveOccurred()) Expect(r).ToNot(BeNil()) - Expect(Unpack(r, tempDir)).To(Succeed()) + details, err := Unpack(r, tempDir) + Expect(details).ToNot(BeNil()) + Expect(err).ToNot(HaveOccurred()) + Expect(filepath.Join(tempDir, "README.md")).To(BeAnExistingFile()) Expect(filepath.Join(tempDir, ".someToolDir", "config.json")).ToNot(BeAnExistingFile()) Expect(filepath.Join(tempDir, "somefile")).To(BeAnExistingFile()) diff --git a/pkg/reconciler/buildrun/resources/sources.go b/pkg/reconciler/buildrun/resources/sources.go index 37e94a7c60..5e7b8b7fee 100644 --- a/pkg/reconciler/buildrun/resources/sources.go +++ b/pkg/reconciler/buildrun/resources/sources.go @@ -5,6 +5,12 @@ package resources import ( + "strconv" + "strings" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + buildv1alpha1 "github.com/shipwright-io/build/pkg/apis/build/v1alpha1" "github.com/shipwright-io/build/pkg/config" "github.com/shipwright-io/build/pkg/reconciler/buildrun/resources/sources" @@ -14,6 +20,8 @@ import ( const defaultSourceName = "default" +const sourceTimestampName = "source-timestamp" + // isLocalCopyBuildSource appends all "Sources" in a single slice, and if any entry is typed // "LocalCopy" it returns first LocalCopy typed BuildSource found, or nil. func isLocalCopyBuildSource( @@ -33,6 +41,15 @@ func isLocalCopyBuildSource( return nil } +func appendSourceTimestampResult(taskSpec *pipelineapi.TaskSpec) { + taskSpec.Results = append(taskSpec.Results, + pipelineapi.TaskResult{ + Name: sources.TaskResultName(defaultSourceName, sourceTimestampName), + Description: "The timestamp of the source.", + }, + ) +} + // AmendTaskSpecWithSources adds the necessary steps to either wait for user upload ("LocalCopy"), or // alternatively, configures the Task steps to use bundle and "git clone". func AmendTaskSpecWithSources( @@ -47,8 +64,10 @@ func AmendTaskSpecWithSources( // create the step for spec.source, either Git or Bundle switch { case build.Spec.Source.BundleContainer != nil: + appendSourceTimestampResult(taskSpec) sources.AppendBundleStep(cfg, taskSpec, build.Spec.Source, defaultSourceName) case build.Spec.Source.URL != nil: + appendSourceTimestampResult(taskSpec) sources.AppendGitStep(cfg, taskSpec, build.Spec.Source, defaultSourceName) } } @@ -65,6 +84,7 @@ func AmendTaskSpecWithSources( func updateBuildRunStatusWithSourceResult(buildrun *buildv1alpha1.BuildRun, results []pipelineapi.TaskRunResult) { buildSpec := buildrun.Status.BuildSpec + // no results for HTTP sources yet switch { case buildSpec.Source.BundleContainer != nil: sources.AppendBundleResult(buildrun, defaultSourceName, results) @@ -73,5 +93,13 @@ func updateBuildRunStatusWithSourceResult(buildrun *buildv1alpha1.BuildRun, resu sources.AppendGitResult(buildrun, defaultSourceName, results) } - // no results for HTTP sources yet + if sourceTimestamp := sources.FindResultValue(results, defaultSourceName, sourceTimestampName); strings.TrimSpace(sourceTimestamp) != "" { + if sec, err := strconv.ParseInt(sourceTimestamp, 10, 64); err == nil { + for i := range buildrun.Status.Sources { + if buildrun.Status.Sources[i].Name == defaultSourceName { + buildrun.Status.Sources[i].Timestamp = &metav1.Time{Time: time.Unix(sec, 0)} + } + } + } + } } diff --git a/pkg/reconciler/buildrun/resources/sources/bundle.go b/pkg/reconciler/buildrun/resources/sources/bundle.go index 8b7eda6402..1d4d52a9bd 100644 --- a/pkg/reconciler/buildrun/resources/sources/bundle.go +++ b/pkg/reconciler/buildrun/resources/sources/bundle.go @@ -8,7 +8,7 @@ import ( "fmt" "strings" - core "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" build "github.com/shipwright-io/build/pkg/apis/build/v1alpha1" "github.com/shipwright-io/build/pkg/config" @@ -24,10 +24,12 @@ func AppendBundleStep( name string, ) { // append the result - taskSpec.Results = append(taskSpec.Results, pipelineapi.TaskResult{ - Name: fmt.Sprintf("%s-source-%s-image-digest", prefixParamsResultsVolumes, name), - Description: "The digest of the bundle image.", - }) + taskSpec.Results = append(taskSpec.Results, + pipelineapi.TaskResult{ + Name: fmt.Sprintf("%s-source-%s-image-digest", PrefixParamsResultsVolumes, name), + Description: "The digest of the bundle image.", + }, + ) // initialize the step from the template and the build-specific arguments bundleStep := pipelineapi.Step{ @@ -37,8 +39,9 @@ func AppendBundleStep( Command: cfg.BundleContainerTemplate.Command, Args: []string{ "--image", source.BundleContainer.Image, - "--target", fmt.Sprintf("$(params.%s-%s)", prefixParamsResultsVolumes, paramSourceRoot), - "--result-file-image-digest", fmt.Sprintf("$(results.%s-source-%s-image-digest.path)", prefixParamsResultsVolumes, name), + "--target", fmt.Sprintf("$(params.%s-%s)", PrefixParamsResultsVolumes, paramSourceRoot), + "--result-file-image-digest", fmt.Sprintf("$(results.%s-source-%s-image-digest.path)", PrefixParamsResultsVolumes, name), + "--result-file-source-timestamp", fmt.Sprintf("$(results.%s-source-%s-source-timestamp.path)", PrefixParamsResultsVolumes, name), }, Env: cfg.BundleContainerTemplate.Env, ComputeResources: cfg.BundleContainerTemplate.Resources, @@ -50,10 +53,10 @@ func AppendBundleStep( if source.Credentials != nil { AppendSecretVolume(taskSpec, source.Credentials.Name) - secretMountPath := fmt.Sprintf("/workspace/%s-pull-secret", prefixParamsResultsVolumes) + secretMountPath := fmt.Sprintf("/workspace/%s-pull-secret", PrefixParamsResultsVolumes) // define the volume mount on the container - bundleStep.VolumeMounts = append(bundleStep.VolumeMounts, core.VolumeMount{ + bundleStep.VolumeMounts = append(bundleStep.VolumeMounts, corev1.VolumeMount{ Name: SanitizeVolumeNameForSecretName(source.Credentials.Name), MountPath: secretMountPath, ReadOnly: true, @@ -75,7 +78,7 @@ func AppendBundleStep( // AppendBundleResult append bundle source result to build run func AppendBundleResult(buildRun *build.BuildRun, name string, results []pipelineapi.TaskRunResult) { - imageDigest := findResultValue(results, fmt.Sprintf("%s-source-%s-image-digest", prefixParamsResultsVolumes, name)) + imageDigest := FindResultValue(results, name, "image-digest") if strings.TrimSpace(imageDigest) != "" { buildRun.Status.Sources = append(buildRun.Status.Sources, build.SourceResult{ diff --git a/pkg/reconciler/buildrun/resources/sources/git.go b/pkg/reconciler/buildrun/resources/sources/git.go index 640e2c7af2..8f310f258f 100644 --- a/pkg/reconciler/buildrun/resources/sources/git.go +++ b/pkg/reconciler/buildrun/resources/sources/git.go @@ -8,10 +8,12 @@ import ( "fmt" "strings" + corev1 "k8s.io/api/core/v1" + buildv1alpha1 "github.com/shipwright-io/build/pkg/apis/build/v1alpha1" "github.com/shipwright-io/build/pkg/config" + pipelineapi "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1" - corev1 "k8s.io/api/core/v1" ) const ( @@ -28,16 +30,20 @@ func AppendGitStep( name string, ) { // append the result - taskSpec.Results = append(taskSpec.Results, pipelineapi.TaskResult{ - Name: fmt.Sprintf("%s-source-%s-%s", prefixParamsResultsVolumes, name, commitSHAResult), - Description: "The commit SHA of the cloned source.", - }, pipelineapi.TaskResult{ - Name: fmt.Sprintf("%s-source-%s-%s", prefixParamsResultsVolumes, name, commitAuthorResult), - Description: "The author of the last commit of the cloned source.", - }, pipelineapi.TaskResult{ - Name: fmt.Sprintf("%s-source-%s-%s", prefixParamsResultsVolumes, name, branchName), - Description: "The name of the branch used of the cloned source.", - }) + taskSpec.Results = append(taskSpec.Results, + pipelineapi.TaskResult{ + Name: fmt.Sprintf("%s-source-%s-%s", PrefixParamsResultsVolumes, name, commitSHAResult), + Description: "The commit SHA of the cloned source.", + }, + pipelineapi.TaskResult{ + Name: fmt.Sprintf("%s-source-%s-%s", PrefixParamsResultsVolumes, name, commitAuthorResult), + Description: "The author of the last commit of the cloned source.", + }, + pipelineapi.TaskResult{ + Name: fmt.Sprintf("%s-source-%s-%s", PrefixParamsResultsVolumes, name, branchName), + Description: "The name of the branch used of the cloned source.", + }, + ) // initialize the step from the template and the build-specific arguments gitStep := pipelineapi.Step{ @@ -46,20 +52,14 @@ func AppendGitStep( ImagePullPolicy: cfg.GitContainerTemplate.ImagePullPolicy, Command: cfg.GitContainerTemplate.Command, Args: []string{ - "--url", - *source.URL, - "--target", - fmt.Sprintf("$(params.%s-%s)", prefixParamsResultsVolumes, paramSourceRoot), - "--result-file-commit-sha", - fmt.Sprintf("$(results.%s-source-%s-%s.path)", prefixParamsResultsVolumes, name, commitSHAResult), - "--result-file-commit-author", - fmt.Sprintf("$(results.%s-source-%s-%s.path)", prefixParamsResultsVolumes, name, commitAuthorResult), - "--result-file-branch-name", - fmt.Sprintf("$(results.%s-source-%s-%s.path)", prefixParamsResultsVolumes, name, branchName), - "--result-file-error-message", - fmt.Sprintf("$(results.%s-error-message.path)", prefixParamsResultsVolumes), - "--result-file-error-reason", - fmt.Sprintf("$(results.%s-error-reason.path)", prefixParamsResultsVolumes), + "--url", *source.URL, + "--target", fmt.Sprintf("$(params.%s-%s)", PrefixParamsResultsVolumes, paramSourceRoot), + "--result-file-commit-sha", fmt.Sprintf("$(results.%s-source-%s-%s.path)", PrefixParamsResultsVolumes, name, commitSHAResult), + "--result-file-commit-author", fmt.Sprintf("$(results.%s-source-%s-%s.path)", PrefixParamsResultsVolumes, name, commitAuthorResult), + "--result-file-branch-name", fmt.Sprintf("$(results.%s-source-%s-%s.path)", PrefixParamsResultsVolumes, name, branchName), + "--result-file-error-message", fmt.Sprintf("$(results.%s-error-message.path)", PrefixParamsResultsVolumes), + "--result-file-error-reason", fmt.Sprintf("$(results.%s-error-reason.path)", PrefixParamsResultsVolumes), + "--result-file-source-timestamp", fmt.Sprintf("$(results.%s-source-%s-source-timestamp.path)", PrefixParamsResultsVolumes, name), }, Env: cfg.GitContainerTemplate.Env, ComputeResources: cfg.GitContainerTemplate.Resources, @@ -86,7 +86,7 @@ func AppendGitStep( // ensure the value is there AppendSecretVolume(taskSpec, source.Credentials.Name) - secretMountPath := fmt.Sprintf("/workspace/%s-source-secret", prefixParamsResultsVolumes) + secretMountPath := fmt.Sprintf("/workspace/%s-source-secret", PrefixParamsResultsVolumes) // define the volume mount on the container gitStep.VolumeMounts = append(gitStep.VolumeMounts, corev1.VolumeMount{ @@ -109,9 +109,9 @@ func AppendGitStep( // AppendGitResult append git source result to build run func AppendGitResult(buildRun *buildv1alpha1.BuildRun, name string, results []pipelineapi.TaskRunResult) { - commitAuthor := findResultValue(results, fmt.Sprintf("%s-source-%s-%s", prefixParamsResultsVolumes, name, commitAuthorResult)) - commitSha := findResultValue(results, fmt.Sprintf("%s-source-%s-%s", prefixParamsResultsVolumes, name, commitSHAResult)) - branchName := findResultValue(results, fmt.Sprintf("%s-source-%s-%s", prefixParamsResultsVolumes, name, branchName)) + commitAuthor := FindResultValue(results, name, commitAuthorResult) + commitSha := FindResultValue(results, name, commitSHAResult) + branchName := FindResultValue(results, name, branchName) if strings.TrimSpace(commitAuthor) != "" || strings.TrimSpace(commitSha) != "" || strings.TrimSpace(branchName) != "" { buildRun.Status.Sources = append(buildRun.Status.Sources, buildv1alpha1.SourceResult{ diff --git a/pkg/reconciler/buildrun/resources/sources/git_test.go b/pkg/reconciler/buildrun/resources/sources/git_test.go index 2f97e64b57..b3fa413578 100644 --- a/pkg/reconciler/buildrun/resources/sources/git_test.go +++ b/pkg/reconciler/buildrun/resources/sources/git_test.go @@ -47,20 +47,14 @@ var _ = Describe("Git", func() { Expect(taskSpec.Steps[0].Name).To(Equal("source-default")) Expect(taskSpec.Steps[0].Image).To(Equal(cfg.GitContainerTemplate.Image)) Expect(taskSpec.Steps[0].Args).To(Equal([]string{ - "--url", - "https://github.com/shipwright-io/build", - "--target", - "$(params.shp-source-root)", - "--result-file-commit-sha", - "$(results.shp-source-default-commit-sha.path)", - "--result-file-commit-author", - "$(results.shp-source-default-commit-author.path)", - "--result-file-branch-name", - "$(results.shp-source-default-branch-name.path)", - "--result-file-error-message", - "$(results.shp-error-message.path)", - "--result-file-error-reason", - "$(results.shp-error-reason.path)", + "--url", "https://github.com/shipwright-io/build", + "--target", "$(params.shp-source-root)", + "--result-file-commit-sha", "$(results.shp-source-default-commit-sha.path)", + "--result-file-commit-author", "$(results.shp-source-default-commit-author.path)", + "--result-file-branch-name", "$(results.shp-source-default-branch-name.path)", + "--result-file-error-message", "$(results.shp-error-message.path)", + "--result-file-error-reason", "$(results.shp-error-reason.path)", + "--result-file-source-timestamp", "$(results.shp-source-default-source-timestamp.path)", })) }) }) @@ -101,22 +95,15 @@ var _ = Describe("Git", func() { Expect(taskSpec.Steps[0].Name).To(Equal("source-default")) Expect(taskSpec.Steps[0].Image).To(Equal(cfg.GitContainerTemplate.Image)) Expect(taskSpec.Steps[0].Args).To(Equal([]string{ - "--url", - "git@github.com:shipwright-io/build.git", - "--target", - "$(params.shp-source-root)", - "--result-file-commit-sha", - "$(results.shp-source-default-commit-sha.path)", - "--result-file-commit-author", - "$(results.shp-source-default-commit-author.path)", - "--result-file-branch-name", - "$(results.shp-source-default-branch-name.path)", - "--result-file-error-message", - "$(results.shp-error-message.path)", - "--result-file-error-reason", - "$(results.shp-error-reason.path)", - "--secret-path", - "/workspace/shp-source-secret", + "--url", "git@github.com:shipwright-io/build.git", + "--target", "$(params.shp-source-root)", + "--result-file-commit-sha", "$(results.shp-source-default-commit-sha.path)", + "--result-file-commit-author", "$(results.shp-source-default-commit-author.path)", + "--result-file-branch-name", "$(results.shp-source-default-branch-name.path)", + "--result-file-error-message", "$(results.shp-error-message.path)", + "--result-file-error-reason", "$(results.shp-error-reason.path)", + "--result-file-source-timestamp", "$(results.shp-source-default-source-timestamp.path)", + "--secret-path", "/workspace/shp-source-secret", })) Expect(len(taskSpec.Steps[0].VolumeMounts)).To(Equal(1)) Expect(taskSpec.Steps[0].VolumeMounts[0].Name).To(Equal("shp-a-secret")) diff --git a/pkg/reconciler/buildrun/resources/sources/http.go b/pkg/reconciler/buildrun/resources/sources/http.go index 28bbf1805e..fbc55f8d1f 100644 --- a/pkg/reconciler/buildrun/resources/sources/http.go +++ b/pkg/reconciler/buildrun/resources/sources/http.go @@ -29,7 +29,7 @@ func AppendHTTPStep( httpStep := pipelineapi.Step{ Name: RemoteArtifactsContainerName, Image: cfg.RemoteArtifactsContainerImage, - WorkingDir: fmt.Sprintf("$(params.%s-%s)", prefixParamsResultsVolumes, paramSourceRoot), + WorkingDir: fmt.Sprintf("$(params.%s-%s)", PrefixParamsResultsVolumes, paramSourceRoot), Command: []string{ "/bin/sh", }, diff --git a/pkg/reconciler/buildrun/resources/sources/utils.go b/pkg/reconciler/buildrun/resources/sources/utils.go index b6b600d97a..2c88041880 100644 --- a/pkg/reconciler/buildrun/resources/sources/utils.go +++ b/pkg/reconciler/buildrun/resources/sources/utils.go @@ -15,7 +15,7 @@ import ( ) const ( - prefixParamsResultsVolumes = "shp" + PrefixParamsResultsVolumes = "shp" paramSourceRoot = "source-root" ) @@ -56,7 +56,7 @@ func AppendSecretVolume( // SanitizeVolumeNameForSecretName creates the name of a Volume for a Secret func SanitizeVolumeNameForSecretName(secretName string) string { // remove forbidden characters - sanitizedName := dnsLabel1123Forbidden.ReplaceAllString(fmt.Sprintf("%s-%s", prefixParamsResultsVolumes, secretName), "-") + sanitizedName := dnsLabel1123Forbidden.ReplaceAllString(fmt.Sprintf("%s-%s", PrefixParamsResultsVolumes, secretName), "-") // ensure maximum length if len(sanitizedName) > 63 { @@ -69,7 +69,16 @@ func SanitizeVolumeNameForSecretName(secretName string) string { return sanitizedName } -func findResultValue(results []pipelineapi.TaskRunResult, name string) string { +func TaskResultName(sourceName, resultName string) string { + return fmt.Sprintf("%s-source-%s-%s", + PrefixParamsResultsVolumes, + sourceName, + resultName, + ) +} + +func FindResultValue(results []pipelineapi.TaskRunResult, sourceName, resultName string) string { + var name = TaskResultName(sourceName, resultName) for _, result := range results { if result.Name == name { return result.Value.StringVal diff --git a/pkg/reconciler/buildrun/resources/taskrun_test.go b/pkg/reconciler/buildrun/resources/taskrun_test.go index 3a4f21a5fd..cb27594b98 100644 --- a/pkg/reconciler/buildrun/resources/taskrun_test.go +++ b/pkg/reconciler/buildrun/resources/taskrun_test.go @@ -84,20 +84,14 @@ var _ = Describe("GenerateTaskrun", func() { Expect(got.Steps[0].Name).To(Equal("source-default")) Expect(got.Steps[0].Command[0]).To(Equal("/ko-app/git")) Expect(got.Steps[0].Args).To(Equal([]string{ - "--url", - *build.Spec.Source.URL, - "--target", - "$(params.shp-source-root)", - "--result-file-commit-sha", - "$(results.shp-source-default-commit-sha.path)", - "--result-file-commit-author", - "$(results.shp-source-default-commit-author.path)", - "--result-file-branch-name", - "$(results.shp-source-default-branch-name.path)", - "--result-file-error-message", - "$(results.shp-error-message.path)", - "--result-file-error-reason", - "$(results.shp-error-reason.path)", + "--url", *build.Spec.Source.URL, + "--target", "$(params.shp-source-root)", + "--result-file-commit-sha", "$(results.shp-source-default-commit-sha.path)", + "--result-file-commit-author", "$(results.shp-source-default-commit-author.path)", + "--result-file-branch-name", "$(results.shp-source-default-branch-name.path)", + "--result-file-error-message", "$(results.shp-error-message.path)", + "--result-file-error-reason", "$(results.shp-error-reason.path)", + "--result-file-source-timestamp", "$(results.shp-source-default-source-timestamp.path)", })) }) diff --git a/test/e2e/v1alpha1/validators_test.go b/test/e2e/v1alpha1/validators_test.go index d85acc33d7..4ec185782b 100644 --- a/test/e2e/v1alpha1/validators_test.go +++ b/test/e2e/v1alpha1/validators_test.go @@ -173,6 +173,9 @@ func validateBuildRunResultsFromGitSource(testBuildRun *buildv1alpha1.BuildRun) Expect(result.Value.StringVal).To(Equal(testBuildRun.Status.Sources[0].Git.CommitAuthor)) case "shp-source-default-branch-name": Expect(result.Value.StringVal).To(Equal(testBuildRun.Status.Sources[0].Git.BranchName)) + case "shp-source-default-source-timestamp": + Expect(strconv.ParseInt(result.Value.StringVal, 10, 64)). + To(Equal(testBuildRun.Status.Sources[0].Timestamp.Unix())) case "shp-image-digest": Expect(result.Value.StringVal).To(Equal(testBuildRun.Status.Output.Digest)) case "shp-image-size": diff --git a/test/e2e/v1beta1/validators_test.go b/test/e2e/v1beta1/validators_test.go index 135df35096..ca1221548c 100644 --- a/test/e2e/v1beta1/validators_test.go +++ b/test/e2e/v1beta1/validators_test.go @@ -172,6 +172,9 @@ func validateBuildRunResultsFromGitSource(testBuildRun *buildv1beta1.BuildRun) { Expect(result.Value.StringVal).To(Equal(testBuildRun.Status.Source.Git.CommitAuthor)) case "shp-source-default-branch-name": Expect(result.Value.StringVal).To(Equal(testBuildRun.Status.Source.Git.BranchName)) + case "shp-source-default-source-timestamp": + Expect(strconv.ParseInt(result.Value.StringVal, 10, 64)). + To(Equal(testBuildRun.Status.Source.Timestamp.Unix())) case "shp-image-digest": Expect(result.Value.StringVal).To(Equal(testBuildRun.Status.Output.Digest)) case "shp-image-size": diff --git a/test/integration/build_to_buildruns_test.go b/test/integration/build_to_buildruns_test.go index ccae8c58a2..104441f8f8 100644 --- a/test/integration/build_to_buildruns_test.go +++ b/test/integration/build_to_buildruns_test.go @@ -39,6 +39,7 @@ var _ = Describe("Integration tests Build and BuildRuns", func() { Expect(err).To(BeNil()) }) + // Delete the ClusterBuildStrategies after each test case AfterEach(func() { diff --git a/test/integration/build_to_git_test.go b/test/integration/build_to_git_test.go index 0f0cefdc80..71add4753b 100644 --- a/test/integration/build_to_git_test.go +++ b/test/integration/build_to_git_test.go @@ -7,10 +7,12 @@ package integration_test import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/shipwright-io/build/pkg/apis/build/v1alpha1" - test "github.com/shipwright-io/build/test/v1alpha1_samples" + corev1 "k8s.io/api/core/v1" "k8s.io/utils/pointer" + + "github.com/shipwright-io/build/pkg/apis/build/v1alpha1" + test "github.com/shipwright-io/build/test/v1alpha1_samples" ) var _ = Describe("Integration tests Build and referenced Source url", func() { diff --git a/test/integration/build_to_taskruns_test.go b/test/integration/build_to_taskruns_test.go index 6766cfcd3b..2279500594 100644 --- a/test/integration/build_to_taskruns_test.go +++ b/test/integration/build_to_taskruns_test.go @@ -31,6 +31,7 @@ var _ = Describe("Integration tests Build and TaskRun", func() { err = tb.CreateClusterBuildStrategy(cbsObject) Expect(err).To(BeNil()) }) + // Delete the ClusterBuildStrategies after each test case AfterEach(func() { _, err = tb.GetBuild(buildObject.Name) diff --git a/test/integration/buildrun_status_test.go b/test/integration/buildrun_status_test.go new file mode 100644 index 0000000000..597da17f00 --- /dev/null +++ b/test/integration/buildrun_status_test.go @@ -0,0 +1,96 @@ +// Copyright The Shipwright Contributors +// +// SPDX-License-Identifier: Apache-2.0 + +package integration_test + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/shipwright-io/build/pkg/apis/build/v1alpha1" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/utils/pointer" +) + +var _ = Describe("Checking BuildRun Status fields", func() { + Context("Verifying BuildRun status source results", func() { + var ( + strategyName string + buildRunName string + ) + + BeforeEach(func() { + id := rand.String(5) + strategyName = fmt.Sprintf("cbs-%s", id) + buildRunName = fmt.Sprintf("buildrun-%s", id) + }) + + AfterEach(func() { + tb.DeleteBR(buildRunName) + tb.DeleteClusterBuildStrategy(strategyName) + }) + + It("should have the correct source timestamp for Git sources", func() { + // Use an empty strategy to only have the source step + strategy := tb.Catalog.ClusterBuildStrategy(strategyName) + Expect(tb.CreateClusterBuildStrategy(strategy)).To(Succeed()) + + // Setup BuildRun with fixed revision where we know the commit details + Expect(tb.CreateBR(&v1alpha1.BuildRun{ + ObjectMeta: metav1.ObjectMeta{Name: buildRunName}, + Spec: v1alpha1.BuildRunSpec{ + BuildSpec: &v1alpha1.BuildSpec{ + Strategy: v1alpha1.Strategy{Kind: (*v1alpha1.BuildStrategyKind)(&strategy.Kind), Name: strategy.Name}, + Source: v1alpha1.Source{ + URL: pointer.String("https://github.com/shipwright-io/sample-go"), + Revision: pointer.String("v0.1.0"), + }, + }, + }, + })).ToNot(HaveOccurred()) + + buildRun, err := tb.GetBRTillCompletion(buildRunName) + Expect(err).ToNot(HaveOccurred()) + Expect(buildRun).ToNot(BeNil()) + + Expect(buildRun.Status.Sources).ToNot(BeEmpty()) + Expect(buildRun.Status.Sources[0].Timestamp).ToNot(BeNil()) + Expect(buildRun.Status.Sources[0].Timestamp.Time).To(BeTemporally("==", time.Unix(1619426578, 0))) + }) + + It("should have the correct source timestamp for Bundle sources", func() { + // Use an empty strategy to only have the source step + strategy := tb.Catalog.ClusterBuildStrategy(strategyName) + Expect(tb.CreateClusterBuildStrategy(strategy)).To(Succeed()) + + // Setup BuildRun with fixed image sha where we know the timestamp details + Expect(tb.CreateBR(&v1alpha1.BuildRun{ + ObjectMeta: metav1.ObjectMeta{Name: buildRunName}, + Spec: v1alpha1.BuildRunSpec{ + BuildSpec: &v1alpha1.BuildSpec{ + Strategy: v1alpha1.Strategy{Kind: (*v1alpha1.BuildStrategyKind)(&strategy.Kind), Name: strategy.Name}, + Source: v1alpha1.Source{ + BundleContainer: &v1alpha1.BundleContainer{ + Image: "ghcr.io/shipwright-io/sample-go/source-bundle@sha256:9a5e264c19980387b8416e0ffa7460488272fb8a6a56127c657edaa2682daab2", + }, + }, + }, + }, + })).ToNot(HaveOccurred()) + + buildRun, err := tb.GetBRTillCompletion(buildRunName) + Expect(err).ToNot(HaveOccurred()) + Expect(buildRun).ToNot(BeNil()) + + Expect(buildRun.Status.Sources).ToNot(BeEmpty()) + Expect(buildRun.Status.Sources[0].Timestamp).ToNot(BeNil()) + Expect(buildRun.Status.Sources[0].Timestamp.Time).To(BeTemporally("==", time.Unix(1691650396, 0))) + }) + }) +}) diff --git a/test/v1alpha1_samples/catalog.go b/test/v1alpha1_samples/catalog.go index 15a64b0ed5..22b82f0231 100644 --- a/test/v1alpha1_samples/catalog.go +++ b/test/v1alpha1_samples/catalog.go @@ -225,6 +225,9 @@ func (c *Catalog) BuildWithOutputSecret(name string, ns string, secretName strin // ClusterBuildStrategy to support tests func (c *Catalog) ClusterBuildStrategy(name string) *build.ClusterBuildStrategy { return &build.ClusterBuildStrategy{ + TypeMeta: metav1.TypeMeta{ + Kind: string(build.ClusterBuildStrategyKind), + }, ObjectMeta: metav1.ObjectMeta{ Name: name, },