Skip to content
This repository has been archived by the owner on Aug 14, 2020. It is now read-only.

Annotate manifest hash #237

Merged
merged 2 commits into from
Jan 26, 2017
Merged

Annotate manifest hash #237

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bb692e3
annotate images with manifest hash
Jan 20, 2017
065228f
lib: stop downloading images if the caller already has this manifest
Jan 20, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions lib/common/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ const (
AppcDockerParentImageID = "appc.io/docker/parentimageid"
AppcDockerEntrypoint = "appc.io/docker/entrypoint"
AppcDockerCmd = "appc.io/docker/cmd"
AppcDockerManifestHash = "appc.io/docker/manifesthash"
)

const defaultTag = "latest"
Expand Down
18 changes: 12 additions & 6 deletions lib/docker2aci.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,11 @@ import (
// CommonConfig represents the shared configuration options for converting
// Docker images.
type CommonConfig struct {
Squash bool // squash the layers in one file
OutputDir string // where to put the resulting ACI
TmpDir string // directory to use for temporary files
Compression common.Compression // which compression to use for the resulting file(s)
Squash bool // squash the layers in one file
OutputDir string // where to put the resulting ACI
TmpDir string // directory to use for temporary files
Compression common.Compression // which compression to use for the resulting file(s)
CurrentManifestHashes []string // any manifest hashes the caller already has

Info log.Logger
Debug log.Logger
Expand Down Expand Up @@ -141,10 +142,15 @@ type converter struct {

func (c *converter) convert() ([]string, error) {
c.config.Debug.Println("Getting image info...")
ancestry, parsedDockerURL, err := c.backend.GetImageInfo(c.dockerURL)
ancestry, manhash, parsedDockerURL, err := c.backend.GetImageInfo(c.dockerURL)
if err != nil {
return nil, err
}
for _, h := range c.config.CurrentManifestHashes {
if manhash == h {
return nil, nil
}
}

layersOutputDir := c.config.OutputDir
if c.config.Squash {
Expand All @@ -163,7 +169,7 @@ func (c *converter) convert() ([]string, error) {
layerCompression = common.NoCompression
}

aciLayerPaths, aciManifests, err := c.backend.BuildACI(ancestry, parsedDockerURL, layersOutputDir, c.config.TmpDir, layerCompression)
aciLayerPaths, aciManifests, err := c.backend.BuildACI(ancestry, manhash, parsedDockerURL, layersOutputDir, c.config.TmpDir, layerCompression)
if err != nil {
return nil, err
}
Expand Down
26 changes: 16 additions & 10 deletions lib/internal/backend/file/file.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,15 +53,21 @@ func NewFileBackend(file *os.File, debug, info log.Logger) *FileBackend {
}
}

func (lb *FileBackend) GetImageInfo(dockerURL string) ([]string, *common.ParsedDockerURL, error) {
// GetImageInfo, given the url for a docker image, will return the
// following:
// - []string: an ordered list of all layer hashes
// - string: a unique identifier for this image, like a hash of the manifest
// - *common.ParsedDockerURL: a parsed docker URL
// - error: an error if one occurred
func (lb *FileBackend) GetImageInfo(dockerURL string) ([]string, string, *common.ParsedDockerURL, error) {
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this return signature is getting messy, it is not obvious what []string, string is. Can we document or name the return values?

// a missing Docker URL could mean that the file only contains one
// image so it's okay for dockerURL to be blank
var parsedDockerURL *common.ParsedDockerURL
if dockerURL != "" {
var err error
parsedDockerURL, err = common.ParseDockerURL(dockerURL)
if err != nil {
return nil, nil, fmt.Errorf("image provided couldnot be parsed: %v", err)
return nil, "", nil, fmt.Errorf("image provided couldnot be parsed: %v", err)
}
}

Expand All @@ -70,25 +76,25 @@ func (lb *FileBackend) GetImageInfo(dockerURL string) ([]string, *common.ParsedD
name := strings.Split(filepath.Base(lb.file.Name()), ".")[0]
appImageID, ancestry, parsedDockerURL, err := getImageID(lb.file, parsedDockerURL, name, lb.debug)
if err != nil {
return nil, nil, err
return nil, "", nil, err
}

if len(ancestry) == 0 {
ancestry, err = getAncestry(lb.file, appImageID, lb.debug)
if err != nil {
return nil, nil, fmt.Errorf("error getting ancestry: %v", err)
return nil, "", nil, fmt.Errorf("error getting ancestry: %v", err)
}
} else {
// for oci the first image is the config
ancestry = append([]string{appImageID}, ancestry...)
}

return ancestry, parsedDockerURL, nil
return ancestry, appImageID, parsedDockerURL, nil
}

func (lb *FileBackend) BuildACI(layerIDs []string, dockerURL *common.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) {
func (lb *FileBackend) BuildACI(layerIDs []string, manhash string, dockerURL *common.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) {
if strings.Contains(layerIDs[0], ":") {
return lb.BuildACIV22(layerIDs, dockerURL, outputDir, tmpBaseDir, compression)
return lb.BuildACIV22(layerIDs, manhash, dockerURL, outputDir, tmpBaseDir, compression)
}
var aciLayerPaths []string
var aciManifests []*schema.ImageManifest
Expand Down Expand Up @@ -125,7 +131,7 @@ func (lb *FileBackend) BuildACI(layerIDs []string, dockerURL *common.ParsedDocke
defer layerFile.Close()

lb.debug.Println("Generating layer ACI...")
aciPath, manifest, err := internal.GenerateACI(i, layerData, dockerURL, outputDir, layerFile, curPwl, compression, lb.debug)
aciPath, manifest, err := internal.GenerateACI(i, manhash, layerData, dockerURL, outputDir, layerFile, curPwl, compression, lb.debug)
if err != nil {
return nil, nil, fmt.Errorf("error generating ACI: %v", err)
}
Expand All @@ -138,7 +144,7 @@ func (lb *FileBackend) BuildACI(layerIDs []string, dockerURL *common.ParsedDocke
return aciLayerPaths, aciManifests, nil
}

func (lb *FileBackend) BuildACIV22(layerIDs []string, dockerURL *common.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) {
func (lb *FileBackend) BuildACIV22(layerIDs []string, manhash string, dockerURL *common.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) {
if len(layerIDs) < 2 {
return nil, nil, fmt.Errorf("insufficient layers for oci image")
}
Expand Down Expand Up @@ -179,7 +185,7 @@ func (lb *FileBackend) BuildACIV22(layerIDs []string, dockerURL *common.ParsedDo
if i != 0 {
aciPath, manifest, err = internal.GenerateACI22LowerLayer(dockerURL, parts[1], outputDir, layerFile, curPwl, compression)
} else {
aciPath, manifest, err = internal.GenerateACI22TopLayer(dockerURL, &imageConfig, parts[1], outputDir, layerFile, curPwl, compression, aciManifests, lb.debug)
aciPath, manifest, err = internal.GenerateACI22TopLayer(dockerURL, manhash, &imageConfig, parts[1], outputDir, layerFile, curPwl, compression, aciManifests, lb.debug)
}
if err != nil {
return nil, nil, fmt.Errorf("error generating ACI: %v", err)
Expand Down
28 changes: 17 additions & 11 deletions lib/internal/backend/repository/repository.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,10 +87,16 @@ func NewRepositoryBackend(username string, password string, insecure common.Inse
}
}

func (rb *RepositoryBackend) GetImageInfo(url string) ([]string, *common.ParsedDockerURL, error) {
// GetImageInfo, given the url for a docker image, will return the
// following:
// - []string: an ordered list of all layer hashes
// - string: a unique identifier for this image, like a hash of the manifest
// - *common.ParsedDockerURL: a parsed docker URL
// - error: an error if one occurred
func (rb *RepositoryBackend) GetImageInfo(url string) ([]string, string, *common.ParsedDockerURL, error) {
dockerURL, err := common.ParseDockerURL(url)
if err != nil {
return nil, nil, err
return nil, "", nil, err
}

var supportsV2, supportsV1, ok bool
Expand All @@ -100,42 +106,42 @@ func (rb *RepositoryBackend) GetImageInfo(url string) ([]string, *common.ParsedD
var err error
URLSchema, supportsV2, err = rb.supportsRegistry(dockerURL.IndexURL, registryV2)
if err != nil {
return nil, nil, err
return nil, "", nil, err
}
rb.schema = URLSchema + "://"
rb.hostsV2Support[dockerURL.IndexURL] = supportsV2
}

// try v2
if supportsV2 {
layers, dockerURL, err := rb.getImageInfoV2(dockerURL)
layers, manhash, dockerURL, err := rb.getImageInfoV2(dockerURL)
if !isErrHTTP404(err) {
return layers, dockerURL, err
return layers, manhash, dockerURL, err
}
// fallback on 404 failure
rb.hostsV1fallback = true
}

URLSchema, supportsV1, err = rb.supportsRegistry(dockerURL.IndexURL, registryV1)
if err != nil {
return nil, nil, err
return nil, "", nil, err
}
if !supportsV1 && rb.hostsV1fallback {
return nil, nil, fmt.Errorf("attempted fallback to API v1 but not supported")
return nil, "", nil, fmt.Errorf("attempted fallback to API v1 but not supported")
}
if !supportsV1 && !supportsV2 {
return nil, nil, fmt.Errorf("registry doesn't support API v2 nor v1")
return nil, "", nil, fmt.Errorf("registry doesn't support API v2 nor v1")
}
rb.schema = URLSchema + "://"
// try v1, hard fail on failure
return rb.getImageInfoV1(dockerURL)
}

func (rb *RepositoryBackend) BuildACI(layerIDs []string, dockerURL *common.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) {
func (rb *RepositoryBackend) BuildACI(layerIDs []string, manhash string, dockerURL *common.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) {
if rb.hostsV1fallback || !rb.hostsV2Support[dockerURL.IndexURL] {
return rb.buildACIV1(layerIDs, dockerURL, outputDir, tmpBaseDir, compression)
return rb.buildACIV1(layerIDs, manhash, dockerURL, outputDir, tmpBaseDir, compression)
} else {
return rb.buildACIV2(layerIDs, dockerURL, outputDir, tmpBaseDir, compression)
return rb.buildACIV2(layerIDs, manhash, dockerURL, outputDir, tmpBaseDir, compression)
}
}

Expand Down
14 changes: 7 additions & 7 deletions lib/internal/backend/repository/repository1.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,29 +40,29 @@ type RepoData struct {
Cookie []string
}

func (rb *RepositoryBackend) getImageInfoV1(dockerURL *common.ParsedDockerURL) ([]string, *common.ParsedDockerURL, error) {
func (rb *RepositoryBackend) getImageInfoV1(dockerURL *common.ParsedDockerURL) ([]string, string, *common.ParsedDockerURL, error) {
repoData, err := rb.getRepoDataV1(dockerURL.IndexURL, dockerURL.ImageName)
if err != nil {
return nil, nil, fmt.Errorf("error getting repository data: %v", err)
return nil, "", nil, fmt.Errorf("error getting repository data: %v", err)
}

// TODO(iaguis) check more endpoints
appImageID, err := rb.getImageIDFromTagV1(repoData.Endpoints[0], dockerURL.ImageName, dockerURL.Tag, repoData)
if err != nil {
return nil, nil, fmt.Errorf("error getting ImageID from tag %s: %v", dockerURL.Tag, err)
return nil, "", nil, fmt.Errorf("error getting ImageID from tag %s: %v", dockerURL.Tag, err)
}

ancestry, err := rb.getAncestryV1(appImageID, repoData.Endpoints[0], repoData)
if err != nil {
return nil, nil, err
return nil, "", nil, err
}

rb.repoData = repoData

return ancestry, dockerURL, nil
return ancestry, appImageID, dockerURL, nil
}

func (rb *RepositoryBackend) buildACIV1(layerIDs []string, dockerURL *common.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) {
func (rb *RepositoryBackend) buildACIV1(layerIDs []string, manhash string, dockerURL *common.ParsedDockerURL, outputDir string, tmpBaseDir string, compression common.Compression) ([]string, []*schema.ImageManifest, error) {
layerFiles := make([]*os.File, len(layerIDs))
layerDatas := make([]types.DockerImageData, len(layerIDs))

Expand Down Expand Up @@ -121,7 +121,7 @@ func (rb *RepositoryBackend) buildACIV1(layerIDs []string, dockerURL *common.Par

for i := len(layerIDs) - 1; i >= 0; i-- {
rb.debug.Println("Generating layer ACI...")
aciPath, manifest, err := internal.GenerateACI(i, layerDatas[i], dockerURL, outputDir, layerFiles[i], curPwl, compression, rb.debug)
aciPath, manifest, err := internal.GenerateACI(i, manhash, layerDatas[i], dockerURL, outputDir, layerFiles[i], curPwl, compression, rb.debug)
if err != nil {
return nil, nil, fmt.Errorf("error generating ACI: %v", err)
}
Expand Down
Loading