diff --git a/cmd/build/main.go b/cmd/build/main.go index 64dd39d937..d815a6fe13 100644 --- a/cmd/build/main.go +++ b/cmd/build/main.go @@ -13,16 +13,13 @@ import ( "github.com/osbuild/images/internal/cmdutil" "github.com/osbuild/images/pkg/arch" "github.com/osbuild/images/pkg/blueprint" - "github.com/osbuild/images/pkg/container" "github.com/osbuild/images/pkg/distro" "github.com/osbuild/images/pkg/distrofactory" - "github.com/osbuild/images/pkg/dnfjson" "github.com/osbuild/images/pkg/manifest" + "github.com/osbuild/images/pkg/manifestgen" "github.com/osbuild/images/pkg/osbuild" - "github.com/osbuild/images/pkg/ostree" "github.com/osbuild/images/pkg/rhsm/facts" "github.com/osbuild/images/pkg/rpmmd" - "github.com/osbuild/images/pkg/sbom" ) func makeManifest( @@ -59,7 +56,7 @@ func makeManifest( fmt.Fprintf(os.Stderr, "[WARNING]\n%s", strings.Join(warnings, "\n")) } - depsolvedSets, err := depsolve(cacheDir, manifest.GetPackageSetChains(), distribution, archName) + depsolvedSets, err := manifestgen.DefaultDepsolver(cacheDir, manifest.GetPackageSetChains(), distribution, archName) if err != nil { return nil, fmt.Errorf("[ERROR] depsolve failed: %w", err) } @@ -71,12 +68,12 @@ func makeManifest( bp = blueprint.Blueprint(*config.Blueprint) } - containerSpecs, err := resolvePipelineContainers(manifest.GetContainerSourceSpecs(), archName) + containerSpecs, err := manifestgen.DefaultContainerResolver(manifest.GetContainerSourceSpecs(), archName) if err != nil { return nil, fmt.Errorf("[ERROR] container resolution failed: %w", err) } - commitSpecs, err := resolvePipelineCommits(manifest.GetOSTreeSourceSpecs()) + commitSpecs, err := manifestgen.DefaultCommitResolver(manifest.GetOSTreeSourceSpecs()) if err != nil { return nil, fmt.Errorf("[ERROR] ostree commit resolution failed: %w", err) } @@ -89,57 +86,6 @@ func makeManifest( return mf, nil } -func resolveContainers(containers []container.SourceSpec, archName string) ([]container.Spec, error) { - resolver := container.NewResolver(archName) - - for _, c := range containers { - resolver.Add(c) - } - - return resolver.Finish() -} - -func resolvePipelineContainers(containerSources map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error) { - containerSpecs := make(map[string][]container.Spec, len(containerSources)) - for plName, sourceSpecs := range containerSources { - specs, err := resolveContainers(sourceSpecs, archName) - if err != nil { - return nil, err - } - containerSpecs[plName] = specs - } - return containerSpecs, nil -} - -func resolvePipelineCommits(commitSources map[string][]ostree.SourceSpec) (map[string][]ostree.CommitSpec, error) { - commits := make(map[string][]ostree.CommitSpec, len(commitSources)) - for name, commitSources := range commitSources { - commitSpecs := make([]ostree.CommitSpec, len(commitSources)) - for idx, commitSource := range commitSources { - var err error - commitSpecs[idx], err = ostree.Resolve(commitSource) - if err != nil { - return nil, err - } - } - commits[name] = commitSpecs - } - return commits, nil -} - -func depsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string]dnfjson.DepsolveResult, error) { - solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch, d.Name(), cacheDir) - depsolvedSets := make(map[string]dnfjson.DepsolveResult) - for name, pkgSet := range packageSets { - res, err := solver.Depsolve(pkgSet, sbom.StandardTypeNone) - if err != nil { - return nil, err - } - depsolvedSets[name] = *res - } - return depsolvedSets, nil -} - func save(ms manifest.OSBuildManifest, fpath string) error { b, err := json.MarshalIndent(ms, "", " ") if err != nil { diff --git a/cmd/gen-manifests/main.go b/cmd/gen-manifests/main.go index ca55a5be6f..e394dff915 100644 --- a/cmd/gen-manifests/main.go +++ b/cmd/gen-manifests/main.go @@ -27,10 +27,10 @@ import ( "github.com/osbuild/images/pkg/distrofactory" "github.com/osbuild/images/pkg/dnfjson" "github.com/osbuild/images/pkg/manifest" + "github.com/osbuild/images/pkg/manifestgen" "github.com/osbuild/images/pkg/ostree" "github.com/osbuild/images/pkg/rhsm/facts" "github.com/osbuild/images/pkg/rpmmd" - "github.com/osbuild/images/pkg/sbom" testrepos "github.com/osbuild/images/test/data/repositories" ) @@ -216,7 +216,7 @@ func makeManifestJob( var depsolvedSets map[string]dnfjson.DepsolveResult if content["packages"] { - depsolvedSets, err = depsolve(cacheDir, manifest.GetPackageSetChains(), distribution, archName) + depsolvedSets, err = manifestgen.DefaultDepsolver(cacheDir, manifest.GetPackageSetChains(), distribution, archName) if err != nil { err = fmt.Errorf("[%s] depsolve failed: %s", filename, err.Error()) return @@ -233,7 +233,7 @@ func makeManifestJob( var containerSpecs map[string][]container.Spec if content["containers"] { - containerSpecs, err = resolvePipelineContainers(manifest.GetContainerSourceSpecs(), archName) + containerSpecs, err = manifestgen.DefaultContainerResolver(manifest.GetContainerSourceSpecs(), archName) if err != nil { return fmt.Errorf("[%s] container resolution failed: %s", filename, err.Error()) } @@ -243,7 +243,7 @@ func makeManifestJob( var commitSpecs map[string][]ostree.CommitSpec if content["commits"] { - commitSpecs, err = resolvePipelineCommits(manifest.GetOSTreeSourceSpecs()) + commitSpecs, err = manifestgen.DefaultCommitResolver(manifest.GetOSTreeSourceSpecs()) if err != nil { return fmt.Errorf("[%s] ostree commit resolution failed: %s", filename, err.Error()) } @@ -269,28 +269,6 @@ func makeManifestJob( return job } -func resolveContainers(containers []container.SourceSpec, archName string) ([]container.Spec, error) { - resolver := container.NewResolver(archName) - - for _, c := range containers { - resolver.Add(c) - } - - return resolver.Finish() -} - -func resolvePipelineContainers(containerSources map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error) { - containerSpecs := make(map[string][]container.Spec, len(containerSources)) - for plName, sourceSpecs := range containerSources { - specs, err := resolveContainers(sourceSpecs, archName) - if err != nil { - return nil, err - } - containerSpecs[plName] = specs - } - return containerSpecs, nil -} - func mockResolveContainers(containerSources map[string][]container.SourceSpec) map[string][]container.Spec { containerSpecs := make(map[string][]container.Spec, len(containerSources)) for plName, sourceSpecs := range containerSources { @@ -318,22 +296,6 @@ func mockResolveContainers(containerSources map[string][]container.SourceSpec) m return containerSpecs } -func resolvePipelineCommits(commitSources map[string][]ostree.SourceSpec) (map[string][]ostree.CommitSpec, error) { - commits := make(map[string][]ostree.CommitSpec, len(commitSources)) - for name, commitSources := range commitSources { - commitSpecs := make([]ostree.CommitSpec, len(commitSources)) - for idx, commitSource := range commitSources { - var err error - commitSpecs[idx], err = ostree.Resolve(commitSource) - if err != nil { - return nil, err - } - } - commits[name] = commitSpecs - } - return commits, nil -} - func mockResolveCommits(commitSources map[string][]ostree.SourceSpec) map[string][]ostree.CommitSpec { commits := make(map[string][]ostree.CommitSpec, len(commitSources)) for name, commitSources := range commitSources { @@ -346,19 +308,6 @@ func mockResolveCommits(commitSources map[string][]ostree.SourceSpec) map[string return commits } -func depsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string]dnfjson.DepsolveResult, error) { - solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch, d.Name(), cacheDir) - depsolvedSets := make(map[string]dnfjson.DepsolveResult) - for name, pkgSet := range packageSets { - res, err := solver.Depsolve(pkgSet, sbom.StandardTypeNone) - if err != nil { - return nil, err - } - depsolvedSets[name] = *res - } - return depsolvedSets, nil -} - func mockDepsolve(packageSets map[string][]rpmmd.PackageSet, repos []rpmmd.RepoConfig, archName string) map[string]dnfjson.DepsolveResult { depsolvedSets := make(map[string]dnfjson.DepsolveResult) diff --git a/pkg/manifestgen/manifestgen.go b/pkg/manifestgen/manifestgen.go new file mode 100644 index 0000000000..865eb73230 --- /dev/null +++ b/pkg/manifestgen/manifestgen.go @@ -0,0 +1,279 @@ +package manifestgen + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/osbuild/images/pkg/blueprint" + "github.com/osbuild/images/pkg/container" + "github.com/osbuild/images/pkg/distro" + "github.com/osbuild/images/pkg/dnfjson" + "github.com/osbuild/images/pkg/manifest" + "github.com/osbuild/images/pkg/osbuild" + "github.com/osbuild/images/pkg/ostree" + "github.com/osbuild/images/pkg/reporegistry" + "github.com/osbuild/images/pkg/rpmmd" + "github.com/osbuild/images/pkg/sbom" +) + +const ( + defaultDepsolverSBOMType = sbom.StandardTypeSpdx + defaultSBOMExt = "spdx.json" + + defaultDepsolveCacheDir = "osbuild-depsolve-dnf" +) + +// Options contains the optional settings for the manifest generation. +// For unset values defaults will be used. +type Options struct { + Cachedir string + // Output is the writer that the generated osbuild manifest will + // written to. + Output io.Writer + + RpmDownloader osbuild.RpmDownloader + + // SBOMWriter will be called for each generated SBOM the + // filename contains the suggest filename string and the + // content can be read + SBOMWriter SBOMWriterFunc + + // CustomSeed overrides the default rng seed, this is mostly + // useful for testing + CustomSeed *int64 + + // Custom "solver" functions, if unset the defaults will be + // used. Only needed for specialized use-cases. + Depsolver DepsolveFunc + ContainerResolver ContainerResolverFunc + CommitResolver CommitResolverFunc +} + +// Generator can generate an osbuild manifest from a given repository +// and options. +type Generator struct { + cacheDir string + out io.Writer + + depsolver DepsolveFunc + containerResolver ContainerResolverFunc + commitResolver CommitResolverFunc + sbomWriter SBOMWriterFunc + + reporegistry *reporegistry.RepoRegistry + + rpmDownloader osbuild.RpmDownloader + + customSeed *int64 +} + +// New will create a new manifest generator +func New(reporegistry *reporegistry.RepoRegistry, opts *Options) (*Generator, error) { + if opts == nil { + opts = &Options{} + } + mg := &Generator{ + reporegistry: reporegistry, + + cacheDir: opts.Cachedir, + out: opts.Output, + depsolver: opts.Depsolver, + containerResolver: opts.ContainerResolver, + commitResolver: opts.CommitResolver, + rpmDownloader: opts.RpmDownloader, + sbomWriter: opts.SBOMWriter, + customSeed: opts.CustomSeed, + } + if mg.out == nil { + mg.out = os.Stdout + } + if mg.depsolver == nil { + mg.depsolver = DefaultDepsolver + } + if mg.containerResolver == nil { + mg.containerResolver = DefaultContainerResolver + } + if mg.commitResolver == nil { + mg.commitResolver = DefaultCommitResolver + } + + return mg, nil +} + +// Generate will generate a new manifest for the given distro/imageType/arch +// combination. +func (mg *Generator) Generate(bp *blueprint.Blueprint, dist distro.Distro, imgType distro.ImageType, a distro.Arch, imgOpts *distro.ImageOptions) error { + if imgOpts == nil { + imgOpts = &distro.ImageOptions{} + } + + repos, err := mg.reporegistry.ReposByImageTypeName(dist.Name(), a.Name(), imgType.Name()) + if err != nil { + return err + } + // To support "user" a.k.a. "3rd party" repositories, these + // will have to be added to the repos with + // .PackageSets set to the "payload" pipeline names + // for the given image type, see e.g. distro/rhel/imagetype.go:Manifest() + preManifest, warnings, err := imgType.Manifest(bp, *imgOpts, repos, mg.customSeed) + if err != nil { + return err + } + if len(warnings) > 0 { + // XXX: what can we do here? for things like json output? + // what are these warnings? + return fmt.Errorf("warnings during manifest creation: %v", strings.Join(warnings, "\n")) + } + depsolved, err := mg.depsolver(mg.cacheDir, preManifest.GetPackageSetChains(), dist, a.Name()) + if err != nil { + return err + } + containerSpecs, err := mg.containerResolver(preManifest.GetContainerSourceSpecs(), a.Name()) + if err != nil { + return err + } + commitSpecs, err := mg.commitResolver(preManifest.GetOSTreeSourceSpecs()) + if err != nil { + return err + } + opts := &manifest.SerializeOptions{ + RpmDownloader: mg.rpmDownloader, + } + mf, err := preManifest.Serialize(depsolved, containerSpecs, commitSpecs, opts) + if err != nil { + return err + } + fmt.Fprintf(mg.out, "%s\n", mf) + + if mg.sbomWriter != nil { + // XXX: this is very similar to + // osbuild-composer:jobimpl-osbuild.go, see if code + // can be shared + for plName, depsolvedPipeline := range depsolved { + pipelinePurpose := "unknown" + switch { + case slices.Contains(imgType.PayloadPipelines(), plName): + pipelinePurpose = "image" + case slices.Contains(imgType.BuildPipelines(), plName): + pipelinePurpose = "buildroot" + } + // XXX: sync with image-builder-cli:build.go name generation - can we have a shared helper? + imageName := fmt.Sprintf("%s-%s-%s", dist.Name(), imgType.Name(), a.Name()) + sbomDocOutputFilename := fmt.Sprintf("%s.%s-%s.%s", imageName, pipelinePurpose, plName, defaultSBOMExt) + + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + if err := enc.Encode(depsolvedPipeline.SBOM.Document); err != nil { + return err + } + if err := mg.sbomWriter(sbomDocOutputFilename, &buf, depsolvedPipeline.SBOM.DocType); err != nil { + return err + } + } + } + + return nil +} + +func xdgCacheHome() (string, error) { + xdgCacheHome := os.Getenv("XDG_CACHE_HOME") + if xdgCacheHome != "" { + return xdgCacheHome, nil + } + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + return filepath.Join(home, ".cache"), nil +} + +// DefaultDepsolver provides a default implementation for depsolving. +// It should rarely be necessary to use it directly and will be used +// by default by manifestgen (unless overriden) +func DefaultDepsolver(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string]dnfjson.DepsolveResult, error) { + if cacheDir == "" { + xdgCacheHomeDir, err := xdgCacheHome() + if err != nil { + return nil, err + } + cacheDir = filepath.Join(xdgCacheHomeDir, defaultDepsolveCacheDir) + } + + solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch, d.Name(), cacheDir) + depsolvedSets := make(map[string]dnfjson.DepsolveResult) + for name, pkgSet := range packageSets { + // Always generate Spdx SBOMs for now, this makes the + // default depsolve slightly slower but it means we + // need no extra argument here to select the SBOM + // type. Once we have more types than Spdx of course + // we need to add a option to select the type. + res, err := solver.Depsolve(pkgSet, sbom.StandardTypeSpdx) + if err != nil { + return nil, fmt.Errorf("error depsolving: %w", err) + } + depsolvedSets[name] = *res + } + return depsolvedSets, nil +} + +func resolveContainers(containers []container.SourceSpec, archName string) ([]container.Spec, error) { + resolver := container.NewResolver(archName) + + for _, c := range containers { + resolver.Add(c) + } + + return resolver.Finish() +} + +// DefaultContainersResolve provides a default implementation for +// container resolving. +// It should rarely be necessary to use it directly and will be used +// by default by manifestgen (unless overriden) +func DefaultContainerResolver(containerSources map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error) { + containerSpecs := make(map[string][]container.Spec, len(containerSources)) + for plName, sourceSpecs := range containerSources { + specs, err := resolveContainers(sourceSpecs, archName) + if err != nil { + return nil, fmt.Errorf("error container resolving: %w", err) + } + containerSpecs[plName] = specs + } + return containerSpecs, nil +} + +// DefaultCommitResolver provides a default implementation for +// ostree commit resolving. +// It should rarely be necessary to use it directly and will be used +// by default by manifestgen (unless overriden) +func DefaultCommitResolver(commitSources map[string][]ostree.SourceSpec) (map[string][]ostree.CommitSpec, error) { + commits := make(map[string][]ostree.CommitSpec, len(commitSources)) + for name, commitSources := range commitSources { + commitSpecs := make([]ostree.CommitSpec, len(commitSources)) + for idx, commitSource := range commitSources { + var err error + commitSpecs[idx], err = ostree.Resolve(commitSource) + if err != nil { + return nil, fmt.Errorf("error ostree commit resolving: %w", err) + } + } + commits[name] = commitSpecs + } + return commits, nil +} + +type ( + DepsolveFunc func(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string]dnfjson.DepsolveResult, error) + + ContainerResolverFunc func(containerSources map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error) + + CommitResolverFunc func(commitSources map[string][]ostree.SourceSpec) (map[string][]ostree.CommitSpec, error) + + SBOMWriterFunc func(filename string, content io.Reader, docType sbom.StandardType) error +) diff --git a/pkg/manifestgen/manifestgen_test.go b/pkg/manifestgen/manifestgen_test.go new file mode 100644 index 0000000000..b98e70bc58 --- /dev/null +++ b/pkg/manifestgen/manifestgen_test.go @@ -0,0 +1,332 @@ +package manifestgen_test + +import ( + "bytes" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "strings" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/osbuild/images/pkg/blueprint" + "github.com/osbuild/images/pkg/container" + "github.com/osbuild/images/pkg/distro" + "github.com/osbuild/images/pkg/distrofactory" + "github.com/osbuild/images/pkg/dnfjson" + "github.com/osbuild/images/pkg/imagefilter" + "github.com/osbuild/images/pkg/manifestgen" + "github.com/osbuild/images/pkg/osbuild" + "github.com/osbuild/images/pkg/osbuild/manifesttest" + "github.com/osbuild/images/pkg/ostree" + "github.com/osbuild/images/pkg/rpmmd" + "github.com/osbuild/images/pkg/sbom" + testrepos "github.com/osbuild/images/test/data/repositories" +) + +func init() { + // silence logrus by default, it is quite verbose + logrus.SetLevel(logrus.WarnLevel) +} + +func sha256For(s string) string { + h := sha256.New() + h.Write([]byte(s)) + bs := h.Sum(nil) + return fmt.Sprintf("sha256:%x", bs) +} + +func TestManifestGeneratorDepsolve(t *testing.T) { + repos, err := testrepos.New() + assert.NoError(t, err) + fac := distrofactory.NewDefault() + + filter, err := imagefilter.New(fac, repos) + assert.NoError(t, err) + res, err := filter.Filter("distro:centos-9", "type:qcow2", "arch:x86_64") + assert.NoError(t, err) + assert.Equal(t, 1, len(res)) + + for _, useLibrepo := range []bool{false, true} { + t.Run(fmt.Sprintf("useLibrepo: %v", useLibrepo), func(t *testing.T) { + var rpmDownloader osbuild.RpmDownloader + if useLibrepo { + rpmDownloader = osbuild.RpmDownloaderLibrepo + } + + var osbuildManifest bytes.Buffer + opts := &manifestgen.Options{ + Output: &osbuildManifest, + Depsolver: fakeDepsolve, + CommitResolver: panicCommitResolver, + ContainerResolver: panicContainerResolver, + + RpmDownloader: rpmDownloader, + } + mg, err := manifestgen.New(repos, opts) + assert.NoError(t, err) + assert.NotNil(t, mg) + var bp blueprint.Blueprint + err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, nil) + require.NoError(t, err) + + pipelineNames, err := manifesttest.PipelineNamesFrom(osbuildManifest.Bytes()) + assert.NoError(t, err) + assert.Equal(t, []string{"build", "os", "image", "qcow2"}, pipelineNames) + + // we expect at least a "kernel" package in the manifest, + // sadly the test distro does not really generate much here so we + // need to use this as a canary that resolving happend + // XXX: add testhelper to manifesttest for this + expectedSha256 := sha256For("kernel") + assert.Contains(t, osbuildManifest.String(), expectedSha256) + + assert.Equal(t, strings.Contains(osbuildManifest.String(), "org.osbuild.librepo"), useLibrepo) + }) + } +} + +func TestManifestGeneratorWithOstreeCommit(t *testing.T) { + var osbuildManifest bytes.Buffer + + repos, err := testrepos.New() + assert.NoError(t, err) + + fac := distrofactory.NewDefault() + filter, err := imagefilter.New(fac, repos) + assert.NoError(t, err) + res, err := filter.Filter("distro:centos-9", "type:edge-ami", "arch:x86_64") + assert.NoError(t, err) + assert.Equal(t, 1, len(res)) + + opts := &manifestgen.Options{ + Output: &osbuildManifest, + Depsolver: fakeDepsolve, + CommitResolver: fakeCommitResolver, + ContainerResolver: panicContainerResolver, + } + imageOpts := &distro.ImageOptions{ + OSTree: &ostree.ImageOptions{ + //ImageRef: "latest/1/x86_64/edge", + URL: "http://example.com/", + }, + } + mg, err := manifestgen.New(repos, opts) + assert.NoError(t, err) + assert.NotNil(t, mg) + var bp blueprint.Blueprint + err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, imageOpts) + assert.NoError(t, err) + + pipelineNames, err := manifesttest.PipelineNamesFrom(osbuildManifest.Bytes()) + assert.NoError(t, err) + assert.Equal(t, []string{"build", "ostree-deployment", "image"}, pipelineNames) + + // XXX: add testhelper to manifesttest for this + assert.Contains(t, osbuildManifest.String(), `{"url":"resolved-url-for-centos/9/x86_64/edge"}`) + // we expect at least a "glibc" package in the manifest, + // sadly the test distro does not really generate much here so we + // need to use this as a canary that resolving happend + // XXX: add testhelper to manifesttest for this + expectedSha256 := sha256For("glibc") + assert.Contains(t, osbuildManifest.String(), expectedSha256) +} + +func fakeDepsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string]dnfjson.DepsolveResult, error) { + depsolvedSets := make(map[string]dnfjson.DepsolveResult) + for name, pkgSets := range packageSets { + repoId := fmt.Sprintf("repo_id_%s", name) + var resolvedSet dnfjson.DepsolveResult + for _, pkgSet := range pkgSets { + for _, pkgName := range pkgSet.Include { + resolvedSet.Packages = append(resolvedSet.Packages, rpmmd.PackageSpec{ + Name: pkgName, + Checksum: sha256For(pkgName), + Path: fmt.Sprintf("path/%s.rpm", pkgName), + RepoID: repoId, + }) + resolvedSet.Repos = append(resolvedSet.Repos, rpmmd.RepoConfig{ + Id: repoId, + Metalink: "https://example.com/metalink", + }) + doc, err := sbom.NewDocument(sbom.StandardTypeSpdx, json.RawMessage(fmt.Sprintf(`{"sbom-for":"%s"}`, name))) + if err != nil { + return nil, err + } + resolvedSet.SBOM = doc + } + } + depsolvedSets[name] = resolvedSet + + } + return depsolvedSets, nil +} + +func fakeCommitResolver(commitSources map[string][]ostree.SourceSpec) (map[string][]ostree.CommitSpec, error) { + commits := make(map[string][]ostree.CommitSpec, len(commitSources)) + for name, commitSources := range commitSources { + commitSpecs := make([]ostree.CommitSpec, len(commitSources)) + for idx, commitSource := range commitSources { + commitSpecs[idx] = ostree.CommitSpec{ + URL: fmt.Sprintf("resolved-url-for-%s", commitSource.Ref), + } + } + commits[name] = commitSpecs + } + return commits, nil + +} + +func panicCommitResolver(commitSources map[string][]ostree.SourceSpec) (map[string][]ostree.CommitSpec, error) { + if len(commitSources) > 0 { + panic("panicCommitResolver") + } + return nil, nil +} + +func fakeContainerResolver(containerSources map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error) { + containerSpecs := make(map[string][]container.Spec, len(containerSources)) + for plName, sourceSpecs := range containerSources { + var containers []container.Spec + for _, spec := range sourceSpecs { + containers = append(containers, container.Spec{ + Source: fmt.Sprintf("resolved-cnt-%s", spec.Source), + Digest: "sha256:" + sha256For("digest:"+spec.Source), + ImageID: "sha256:" + sha256For("id:"+spec.Source), + }) + } + containerSpecs[plName] = containers + } + return containerSpecs, nil +} + +func panicContainerResolver(containerSources map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error) { + if len(containerSources) > 0 { + panic("panicContainerResolver") + } + return nil, nil +} + +func TestManifestGeneratorContainers(t *testing.T) { + repos, err := testrepos.New() + assert.NoError(t, err) + fac := distrofactory.NewDefault() + + filter, err := imagefilter.New(fac, repos) + assert.NoError(t, err) + res, err := filter.Filter("distro:centos-9", "type:qcow2", "arch:x86_64") + assert.NoError(t, err) + assert.Equal(t, 1, len(res)) + + var osbuildManifest bytes.Buffer + opts := &manifestgen.Options{ + Output: &osbuildManifest, + Depsolver: fakeDepsolve, + CommitResolver: panicCommitResolver, + ContainerResolver: fakeContainerResolver, + } + mg, err := manifestgen.New(repos, opts) + assert.NoError(t, err) + assert.NotNil(t, mg) + fakeContainerSource := "registry.gitlab.com/redhat/services/products/image-builder/ci/osbuild-composer/fedora-minimal" + bp := blueprint.Blueprint{ + Containers: []blueprint.Container{ + { + Source: fakeContainerSource, + }, + }, + } + err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, nil) + assert.NoError(t, err) + + // container is included + assert.Contains(t, osbuildManifest.String(), "resolved-cnt-"+fakeContainerSource) +} + +func TestManifestGeneratorDepsolveWithSbomWriter(t *testing.T) { + repos, err := testrepos.New() + assert.NoError(t, err) + fac := distrofactory.NewDefault() + + filter, err := imagefilter.New(fac, repos) + assert.NoError(t, err) + res, err := filter.Filter("distro:centos-9", "type:qcow2", "arch:x86_64") + assert.NoError(t, err) + assert.Equal(t, 1, len(res)) + + var osbuildManifest bytes.Buffer + generatedSboms := map[string]string{} + opts := &manifestgen.Options{ + Output: &osbuildManifest, + Depsolver: fakeDepsolve, + CommitResolver: panicCommitResolver, + ContainerResolver: panicContainerResolver, + + SBOMWriter: func(filename string, content io.Reader, docType sbom.StandardType) error { + assert.Equal(t, sbom.StandardTypeSpdx, docType) + + b, err := io.ReadAll(content) + assert.NoError(t, err) + generatedSboms[filename] = strings.TrimSpace(string(b)) + return nil + }, + } + mg, err := manifestgen.New(repos, opts) + assert.NoError(t, err) + assert.NotNil(t, mg) + var bp blueprint.Blueprint + err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, nil) + require.NoError(t, err) + + assert.Contains(t, generatedSboms, "centos-9-qcow2-x86_64.buildroot-build.spdx.json") + assert.Contains(t, generatedSboms, "centos-9-qcow2-x86_64.image-os.spdx.json") + expected := map[string]string{ + "centos-9-qcow2-x86_64.buildroot-build.spdx.json": `{"sbom-for":"build"}`, + "centos-9-qcow2-x86_64.image-os.spdx.json": `{"sbom-for":"os"}`, + } + assert.Equal(t, expected, generatedSboms) +} + +func TestManifestGeneratorSeed(t *testing.T) { + repos, err := testrepos.New() + assert.NoError(t, err) + fac := distrofactory.NewDefault() + + filter, err := imagefilter.New(fac, repos) + assert.NoError(t, err) + res, err := filter.Filter("distro:centos-9", "type:qcow2", "arch:x86_64") + assert.NoError(t, err) + assert.Equal(t, 1, len(res)) + + for _, withCustomSeed := range []bool{false, true} { + var osbuildManifest bytes.Buffer + opts := &manifestgen.Options{ + Output: &osbuildManifest, + Depsolver: fakeDepsolve, + } + if withCustomSeed { + customSeed := int64(123) + opts.CustomSeed = &customSeed + } + + mg, err := manifestgen.New(repos, opts) + assert.NoError(t, err) + + var bp blueprint.Blueprint + err = mg.Generate(&bp, res[0].Distro, res[0].ImgType, res[0].Arch, nil) + assert.NoError(t, err) + + // with the customSeed we always get a predicatable uuid for + // the xfs boot partition + needle := `f1405ced-8b99-48ba-b910-9259515bf702` + if withCustomSeed { + assert.Contains(t, osbuildManifest.String(), needle) + } else { + assert.NotContains(t, osbuildManifest.String(), needle) + } + } +} diff --git a/pkg/osbuild/manifesttest/manifesttest.go b/pkg/osbuild/manifesttest/manifesttest.go new file mode 100644 index 0000000000..543c8f2b3c --- /dev/null +++ b/pkg/osbuild/manifesttest/manifesttest.go @@ -0,0 +1,25 @@ +package manifesttest + +import ( + "encoding/json" + "fmt" +) + +// PipelineNamesFrom will return all pipeline names from an osbuild +// json manifest. It will error on missing pipelines. +func PipelineNamesFrom(osbuildManifest []byte) ([]string, error) { + var manifest map[string]interface{} + + if err := json.Unmarshal(osbuildManifest, &manifest); err != nil { + return nil, fmt.Errorf("cannot unmarshal manifest: %w", err) + } + if manifest["pipelines"] == nil { + return nil, fmt.Errorf("cannot find any pipelines in %v", manifest) + } + pipelines := manifest["pipelines"].([]interface{}) + pipelineNames := make([]string, len(pipelines)) + for idx, pi := range pipelines { + pipelineNames[idx] = pi.(map[string]interface{})["name"].(string) + } + return pipelineNames, nil +} diff --git a/pkg/osbuild/manifesttest/manifesttest_test.go b/pkg/osbuild/manifesttest/manifesttest_test.go new file mode 100644 index 0000000000..8d50c55dfb --- /dev/null +++ b/pkg/osbuild/manifesttest/manifesttest_test.go @@ -0,0 +1,35 @@ +package manifesttest_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/osbuild/images/pkg/osbuild/manifesttest" +) + +var fakeOsbuildManifest = `{ + "version": "2", + "pipelines": [ + { + "name": "noop" + }, + { + "name": "noop2" + } + ] +}` + +func TestPipelineNamesFrom(t *testing.T) { + names, err := manifesttest.PipelineNamesFrom([]byte(fakeOsbuildManifest)) + assert.NoError(t, err) + assert.Equal(t, []string{"noop", "noop2"}, names) +} + +func TestPipelineNamesFromSad(t *testing.T) { + _, err := manifesttest.PipelineNamesFrom([]byte("bad-json")) + assert.ErrorContains(t, err, "cannot unmarshal manifest: invalid char") + + _, err = manifesttest.PipelineNamesFrom([]byte("{}")) + assert.ErrorContains(t, err, "cannot find any pipelines in map[]") +}