Skip to content

Commit

Permalink
feat: support importing sboms along with images (#567)
Browse files Browse the repository at this point in the history 
When we import an image via the from: directive, also pull in the sbom
if there is one from the source registry. Else we need to rebuild the
sbom and there may not be enough state/information to do so.

Signed-off-by: Ramkumar Chinchani <rchincha@cisco.com>
  • Loading branch information
@rchincha
rchincha authored Mar 18, 2024
1 parent b585bfb commit f3a8d3b
Show file tree
Hide file tree
Showing 8 changed files with 603 additions and 276 deletions.
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ BATS = $(TOOLS_D)/bin/bats
BATS_VERSION := v1.10.0
# OCI registry
ZOT := $(TOOLS_D)/bin/zot
ZOT_VERSION := v2.0.0
ZOT_VERSION := v2.0.2

export PATH := $(TOOLS_D)/bin:$(PATH)

Expand Down
10 changes: 9 additions & 1 deletion pkg/stacker/base.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,15 @@ func GetBase(o BaseLayerOpts) error {
case types.OCILayer:
fallthrough
case types.DockerLayer:
return importContainersImage(o.Layer.From, o.Config, o.Progress)
err := importContainersImage(o.Layer.From, o.Config, o.Progress)
if o.Layer.Bom != nil && o.Layer.Bom.Generate && (o.Layer.From.Type == types.DockerLayer) {
bomPath := path.Join(o.Config.StackerDir, "artifacts", o.Name)
err = getArtifact(bomPath, "application/spdx+json", o.Layer.From.Url, "", "", o.Layer.From.Insecure)
if err != nil {
log.Errorf("sbom for image %s not found", o.Layer.From.Url)
}
}
return err
default:
return errors.Errorf("unknown layer type: %v", o.Layer.From.Type)
}
Expand Down
2 changes: 2 additions & 0 deletions pkg/stacker/build.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ type BuildArgs struct {
SetupOnly bool
Progress bool
AnnotationsNamespace string
Username string
Password string
}

// Builder is responsible for building the layers based on stackerfiles
Expand Down
266 changes: 4 additions & 262 deletions pkg/stacker/publisher.go
View file
Original file line number Diff line number Diff line change
@@ -1,23 +1,14 @@
package stacker

import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"regexp"
"strings"

godigest "github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/specs-go"
ispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/opencontainers/umoci"
"github.com/opencontainers/umoci/oci/casext"
"github.com/pkg/errors"
Expand Down Expand Up @@ -75,255 +66,6 @@ func NewPublisher(opts *PublishArgs) *Publisher {
}
}

func basicAuth(username, password string) string {
auth := username + ":" + password
return base64.StdEncoding.EncodeToString([]byte(auth))
}

func clientRequest(method, url, username, password string, headers map[string]string, body io.Reader) (*http.Response, error) {
req, err := http.NewRequestWithContext(context.TODO(), method, url, body)
if err != nil {
log.Errorf("unable to create http request err:%s", err)
return nil, err
}

// FIXME: handle bearer auth also
if username != "" && password != "" {
req.Header.Add("Authorization", "Basic "+basicAuth(username, password))
}

if len(headers) > 0 {
for k, v := range headers {
req.Header.Add(k, v)
}
}

res, err := http.DefaultClient.Do(req)
if err != nil {
log.Errorf("http request failed url:%s", url)
return nil, err
}

return res, nil
}

func fileDigest(path string) (*godigest.Digest, error) {
fh, err := os.Open(path)
if err != nil {
log.Errorf("unable to open file:%s, err:%s", path, err)
return nil, err
}
defer fh.Close()

dgst, err := godigest.FromReader(fh)
if err != nil {
log.Errorf("unable get digest for file:%s, err:%s", path, err)
return nil, err
}

return &dgst, nil
}

// publishArtifact to a registry/repo for this subject
func (p *Publisher) publishArtifact(path, mtype, registry, repo, subjectTag string, skipTLS bool) error {
username := p.opts.Username
password := p.opts.Password

subject := distspecURL(registry, repo, subjectTag, skipTLS)

// check subject exists
res, err := clientRequest(http.MethodHead, subject, username, password, nil, nil)
if err != nil {
log.Errorf("unable to check subject:%s, err:%s", subject, err)
return err
}
if res == nil || res.StatusCode != http.StatusOK {
log.Errorf("subject:%s doesn't exist, ignoring and proceeding", subject)
}

slen := res.ContentLength
smtype := res.Header.Get("Content-Type")
sdgst, err := godigest.Parse(res.Header.Get("Docker-Content-Digest"))
if slen < 0 || smtype == "" || sdgst == "" || err != nil {
log.Errorf("unable to get descriptor details for subject:%s", subject)
return errors.Errorf("unable to get descriptor details for subject:%s", subject)
}

// upload the artifact
finfo, err := os.Lstat(path)
if err != nil {
log.Errorf("unable to stat file:%s, err:%s", path, err)
return err
}

dgst, err := fileDigest(path)
if err != nil {
log.Errorf("unable get digest for file:%s, err:%s", path, err)
return err
}

fh, err := os.Open(path)
if err != nil {
log.Errorf("unable to open file:%s, err:%s", path, err)
return err
}
defer fh.Close()

if err := uploadBlob(registry, repo, path, username, password, fh, finfo.Size(), dgst, skipTLS); err != nil {
log.Errorf("unable to upload file:%s, err:%s", path, err)
return err
}

// check and upload emptyJSON blob
erdr := bytes.NewReader(ispec.DescriptorEmptyJSON.Data)
edgst := ispec.DescriptorEmptyJSON.Digest

if err := uploadBlob(registry, repo, path, username, password, erdr, ispec.DescriptorEmptyJSON.Size, &edgst, skipTLS); err != nil {
log.Errorf("unable to upload file:%s, err:%s", path, err)
return err
}

// upload the reference manifest
manifest := ispec.Manifest{
Versioned: specs.Versioned{
SchemaVersion: 2,
},
MediaType: ispec.MediaTypeImageManifest,
ArtifactType: mtype,
Config: ispec.DescriptorEmptyJSON,
Subject: &ispec.Descriptor{
MediaType: ispec.MediaTypeImageManifest,
Size: slen,
Digest: sdgst,
},
Layers: []ispec.Descriptor{
ispec.Descriptor{
MediaType: mtype,
Size: finfo.Size(),
Digest: *dgst,
},
},
}

//content, err := json.MarshalIndent(&manifest, "", "\t")
content, err := json.Marshal(&manifest)
if err != nil {
log.Errorf("unable to marshal image manifest, err:%s", err)
return err
}

// artifact manifest
var regUrl string
mdgst := godigest.FromBytes(content)
if skipTLS {
regUrl = fmt.Sprintf("http://%s/v2%s/manifests/%s", registry, strings.Split(repo, ":")[0], mdgst.String())
} else {
regUrl = fmt.Sprintf("https://%s/v2%s/manifests/%s", registry, strings.Split(repo, ":")[0], mdgst.String())
}
hdrs := map[string]string{
"Content-Type": ispec.MediaTypeImageManifest,
"Content-Length": fmt.Sprintf("%d", len(content)),
}
res, err = clientRequest(http.MethodPut, regUrl, username, password, hdrs, bytes.NewBuffer(content))
if err != nil {
log.Errorf("unable to check subject:%s, err:%s", subject, err)
return err
}
if res == nil || res.StatusCode != http.StatusCreated {
log.Errorf("unable to upload manifest, url:%s", regUrl)
return errors.Errorf("unable to upload manifest, url:%s", regUrl)
}

log.Infof("Copying artifact '%s' done", path)

return nil
}

func distspecURL(registry, repo, tag string, skipTLS bool) string {
var url string

if skipTLS {
url = fmt.Sprintf("http://%s/v2%s/manifests/%s", registry, strings.Split(repo, ":")[0], tag)
} else {
url = fmt.Sprintf("https://%s/v2%s/manifests/%s", registry, strings.Split(repo, ":")[0], tag)
}

return url
}

func uploadBlob(registry, repo, path, username, password string, reader io.Reader, size int64, dgst *godigest.Digest, skipTLS bool) error {
// upload with POST, PUT sequence
var regUrl string
if skipTLS {
regUrl = fmt.Sprintf("http://%s/v2%s/blobs/%s", registry, strings.Split(repo, ":")[0], dgst.String())
} else {
regUrl = fmt.Sprintf("https://%s/v2%s/blobs/%s", registry, strings.Split(repo, ":")[0], dgst.String())
}

subject := distspecURL(registry, repo, "", skipTLS)

log.Debugf("check blob before upload (HEAD): %s", regUrl)
res, err := clientRequest(http.MethodHead, regUrl, username, password, nil, nil)
if err != nil {
log.Errorf("unable to check blob:%s, err:%s", subject, err)
return err
}
log.Debugf("http response headers: +%v status:%v", res.Header, res.Status)
hdr := res.Header.Get("Docker-Content-Digest")
if hdr != "" {
log.Infof("Copying blob %s skipped: already exists", dgst.Hex()[:12])
return nil
}

if skipTLS {
regUrl = fmt.Sprintf("http://%s/v2%s/blobs/uploads/", registry, strings.Split(repo, ":")[0])
} else {
regUrl = fmt.Sprintf("https://%s/v2%s/blobs/uploads/", registry, strings.Split(repo, ":")[0])
}

log.Debugf("new blob upload (POST): %s", regUrl)
res, err = clientRequest(http.MethodPost, regUrl, username, password, nil, nil)
if err != nil {
log.Errorf("post unable to check subject:%s, err:%s", subject, err)
return err
}
log.Debugf("http response headers: +%v status:%v", res.Header, res.Status)
loc, err := res.Location()
if err != nil {
log.Errorf("unable get upload location url:%s, err:%s", regUrl, err)
return err
}

log.Debugf("finish blob upload (PUT): %s", regUrl)
req, err := http.NewRequestWithContext(context.TODO(), http.MethodPut, loc.String(), reader)
if err != nil {
log.Errorf("unable to create a http request url:%s", subject)
return err
}
if username != "" && password != "" {
req.Header.Add("Authorization", "Basic "+basicAuth(username, password))
}
req.URL.RawQuery = url.Values{
"digest": {dgst.String()},
}.Encode()

req.ContentLength = size

res, err = http.DefaultClient.Do(req)
if err != nil {
log.Errorf("http request failed url:%s", subject)
return err
}
if res == nil || res.StatusCode != http.StatusCreated {
log.Errorf("unable to upload artifact:%s to url:%s", path, regUrl)
return errors.Errorf("unable to upload artifact:%s to url:%s", path, regUrl)
}

log.Infof("Copying blob %s done", dgst.Hex()[:12])

return nil
}

// Publish layers in a single stackerfile
func (p *Publisher) Publish(file string) error {
opts := p.opts
Expand Down Expand Up @@ -452,14 +194,14 @@ func (p *Publisher) Publish(file string) error {
repo := url.Path

// publish sbom
if err := p.publishArtifact(path.Join(opts.Config.StackerDir, "artifacts", layerName, fmt.Sprintf("%s.json", layerName)),
"application/spdx+json", registry, repo, layerTypeTag, opts.SkipTLS); err != nil {
if err := publishArtifact(path.Join(opts.Config.StackerDir, "artifacts", layerName, fmt.Sprintf("%s.json", layerName)),
"application/spdx+json", registry, repo, layerTypeTag, p.opts.Username, p.opts.Password, opts.SkipTLS); err != nil {
return err
}

// publish inventory
if err := p.publishArtifact(path.Join(opts.Config.StackerDir, "artifacts", layerName, "inventory.json"),
"application/vnd.stackerbuild.inventory+json", registry, repo, layerTypeTag, opts.SkipTLS); err != nil {
if err := publishArtifact(path.Join(opts.Config.StackerDir, "artifacts", layerName, "inventory.json"),
"application/vnd.stackerbuild.inventory+json", registry, repo, layerTypeTag, p.opts.Username, p.opts.Password, opts.SkipTLS); err != nil {
return err
}

Expand Down
Loading

0 comments on commit f3a8d3b

Please sign in to comment.