Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for normalizing the archived files metadata #47

Closed
wants to merge 3 commits into from
+186 −32
Closed

Add support for normalizing the archived files metadata #47

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6a2a1fd
Add missing line separator in zip_archiver_test.go
64kramsystem Jul 5, 2019
4ff3294
Add helper API ensureFileChecksum() to zip_archiver_test.go
64kramsystem Jul 5, 2019
347a1af
Add support for normalizing the archived files metadata
64kramsystem Apr 4, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions archive/archiver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@ import (
)

type Archiver interface {
ArchiveContent(content []byte, infilename string) error
ArchiveFile(infilename string) error
ArchiveDir(indirname string, excludes []string) error
ArchiveMultiple(content map[string][]byte) error
ArchiveContent(content []byte, infilename string, normalizeFilesMetadata bool) error
ArchiveFile(infilename string, normalizeFilesMetadata bool) error
ArchiveDir(indirname string, excludes []string, normalizeFilesMetadata bool) error
ArchiveMultiple(content map[string][]byte, normalizeFilesMetadata bool) error
}

type ArchiverBuilder func(filepath string) Archiver
Expand Down
17 changes: 12 additions & 5 deletions archive/data_source_archive_file.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,12 @@ func dataSourceFile() *schema.Resource {
Required: true,
ForceNew: true,
},
"normalize_files_metadata": {
Type: schema.TypeBool,
Optional: true,
ForceNew: true,
Default: false,
},
"source": {
Type: schema.TypeSet,
Optional: true,
Expand Down Expand Up @@ -165,6 +171,7 @@ func expandStringList(configured []interface{}) []string {
func archive(d *schema.ResourceData) error {
archiveType := d.Get("type").(string)
outputPath := d.Get("output_path").(string)
normalizeFilesMetadata := d.Get("normalize_files_metadata").(bool)

archiver := getArchiver(archiveType, outputPath)
if archiver == nil {
Expand All @@ -175,21 +182,21 @@ func archive(d *schema.ResourceData) error {
if excludes, ok := d.GetOk("excludes"); ok {
excludeList := expandStringList(excludes.(*schema.Set).List())

if err := archiver.ArchiveDir(dir.(string), excludeList); err != nil {
if err := archiver.ArchiveDir(dir.(string), excludeList, normalizeFilesMetadata); err != nil {
return fmt.Errorf("error archiving directory: %s", err)
}
} else {
if err := archiver.ArchiveDir(dir.(string), []string{""}); err != nil {
if err := archiver.ArchiveDir(dir.(string), []string{""}, normalizeFilesMetadata); err != nil {
return fmt.Errorf("error archiving directory: %s", err)
}
}
} else if file, ok := d.GetOk("source_file"); ok {
if err := archiver.ArchiveFile(file.(string)); err != nil {
if err := archiver.ArchiveFile(file.(string), normalizeFilesMetadata); err != nil {
return fmt.Errorf("error archiving file: %s", err)
}
} else if filename, ok := d.GetOk("source_content_filename"); ok {
content := d.Get("source_content").(string)
if err := archiver.ArchiveContent([]byte(content), filename.(string)); err != nil {
if err := archiver.ArchiveContent([]byte(content), filename.(string), normalizeFilesMetadata); err != nil {
return fmt.Errorf("error archiving content: %s", err)
}
} else if v, ok := d.GetOk("source"); ok {
Expand All @@ -199,7 +206,7 @@ func archive(d *schema.ResourceData) error {
src := v.(map[string]interface{})
content[src["filename"].(string)] = []byte(src["content"].(string))
}
if err := archiver.ArchiveMultiple(content); err != nil {
if err := archiver.ArchiveMultiple(content, normalizeFilesMetadata); err != nil {
return fmt.Errorf("error archiving content: %s", err)
}
} else {
Expand Down
105 changes: 89 additions & 16 deletions archive/zip_archiver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,18 @@ package archive
import (
"archive/zip"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"sort"
"time"
)

const (
uint32max = (1 << 32) - 1
)

type ZipArchiver struct {
filepath string
filewriter *os.File
Expand All @@ -22,22 +27,35 @@ func NewZipArchiver(filepath string) Archiver {
}
}

func (a *ZipArchiver) ArchiveContent(content []byte, infilename string) error {
func (a *ZipArchiver) ArchiveContent(content []byte, infilename string, normalizeFilesMetadata bool) error {
if err := a.open(); err != nil {
return err
}
defer a.close()

f, err := a.writer.Create(filepath.ToSlash(infilename))
if err != nil {
return err
var f io.Writer
var err error

if normalizeFilesMetadata {
fh := prepareEmptyHeader(content, infilename)
normalizeCompressingFile(fh)

f, err = a.writer.CreateHeader(fh)
if err != nil {
return fmt.Errorf("error creating file inside archive: %s", err)
}
} else {
f, err = a.writer.Create(filepath.ToSlash(infilename))
if err != nil {
return err
}
}

_, err = f.Write(content)
return err
}

func (a *ZipArchiver) ArchiveFile(infilename string) error {
func (a *ZipArchiver) ArchiveFile(infilename string, normalizeFilesMetadata bool) error {
fi, err := assertValidFile(infilename)
if err != nil {
return err
Expand All @@ -58,9 +76,14 @@ func (a *ZipArchiver) ArchiveFile(infilename string) error {
return fmt.Errorf("error creating file header: %s", err)
}
fh.Name = filepath.ToSlash(fi.Name())
fh.Method = zip.Deflate
// fh.Modified alone isn't enough when using a zero value
fh.SetModTime(time.Time{})

if normalizeFilesMetadata {
normalizeCompressingFile(fh)
} else {
fh.Method = zip.Deflate
// fh.Modified alone isn't enough when using a zero value
fh.SetModTime(time.Time{})
}

f, err := a.writer.CreateHeader(fh)
if err != nil {
Expand All @@ -84,7 +107,38 @@ func checkMatch(fileName string, excludes []string) (value bool) {
return false
}

func (a *ZipArchiver) ArchiveDir(indirname string, excludes []string) error {
// The basic file header is very simple. The UncompressedSize logic is not a real-world use case
// in this context, but "640K ought to be enough for anybody".
//
// For reference, see golang/src/archive/zip/struct.go.
func prepareEmptyHeader(content []byte, infilename string) *zip.FileHeader {
fh := &zip.FileHeader{
Name: filepath.ToSlash(infilename),
UncompressedSize64: uint64(len(content)),
}

if fh.UncompressedSize64 > uint32max {
fh.UncompressedSize = uint32max
} else {
fh.UncompressedSize = uint32(fh.UncompressedSize64)
}

return fh
}

// Normalize the fields:
//
// - no compression, so the compressed stream is essentially a copy;
// - fixed date;
// - fixed file permissions.
//
func normalizeCompressingFile(fh *zip.FileHeader) {
fh.Method = zip.Store
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This causes a very sneaky compatibility issue with Java's standard java.util.zip, resulting in:

java.util.zip.ZipException: only DEFLATED entries can have EXT descriptor
        at java.base/java.util.zip.ZipInputStream.readLOC(ZipInputStream.java:311)
        at java.base/java.util.zip.ZipInputStream.getNextEntry(ZipInputStream.java:123)
        at com.meh.test.UnzipFiles.unzip(UnzipFiles.java:30)
        at com.meh.test.UnzipFiles.main(UnzipFiles.java:17)

The quick summary seems to be: Go's zip uses streaming, meaning it first writes the file entry body, and then writes a so-called "ext" data descriptor with size information. Then, java.util.zip can extract these if they use DEFLATE, since DEFLATE encodes data in such a way that you can know when you have all the data. STORE, on the other hand, has no such encoded info, and Java fails to handle it.

Details:

Impact case: The Amazon ECS (Blue/Green) deploy provider of AWS CodePipeline raises an exception if a normalized zip file is fed to it:

Exception while trying to read the task definition artifact file from: <source stage>

We use archive_file to produce the zip, containing appspec.yaml and taskdef.json files for CodeDeploy and ECS, respectively.

AWS services seem to be heavily Java based, so while we don't see the full Java exception output, it seems likely that this is the problem.

fh.SetModTime(time.Date(1981, 4, 10, 0, 0, 0, 0, time.UTC))
fh.SetMode(0644)
Copy link

@bboe bboe Jul 5, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What if the file is executable?

See here for an example: https://github.com/terraform-providers/terraform-provider-archive/pull/41/files#diff-572809c26c1b9c23a665ebaea5dd7ea8R70

Copy link
Author

@64kramsystem 64kramsystem Jul 12, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not familiar with executing binaries inside a Lambda functions; based on a brief research, maybe Lambda resets the permissions of the archived files - see, for example:

It's not 100% clear from those resources where their problem is - it could be the zip library not storing the file permissions (not all do), or Lambda resetting the permissions when unarchiving; if it's the latter case, managing executable bit would not yield any effect.

Do you have direct experience on this use case, so that I know with certainty what's the exact behavior?

}

func (a *ZipArchiver) ArchiveDir(indirname string, excludes []string, normalizeFilesMetadata bool) error {
_, err := assertValidDir(indirname)
if err != nil {
return err
Expand Down Expand Up @@ -128,9 +182,14 @@ func (a *ZipArchiver) ArchiveDir(indirname string, excludes []string) error {
return fmt.Errorf("error creating file header: %s", err)
}
fh.Name = filepath.ToSlash(relname)
fh.Method = zip.Deflate
// fh.Modified alone isn't enough when using a zero value
fh.SetModTime(time.Time{})

if normalizeFilesMetadata {
normalizeCompressingFile(fh)
} else {
fh.Method = zip.Deflate
// fh.Modified alone isn't enough when using a zero value
fh.SetModTime(time.Time{})
}

f, err := a.writer.CreateHeader(fh)
if err != nil {
Expand All @@ -145,7 +204,7 @@ func (a *ZipArchiver) ArchiveDir(indirname string, excludes []string) error {
})
}

func (a *ZipArchiver) ArchiveMultiple(content map[string][]byte) error {
func (a *ZipArchiver) ArchiveMultiple(content map[string][]byte, normalizeFilesMetadata bool) error {
if err := a.open(); err != nil {
return err
}
Expand All @@ -161,10 +220,24 @@ func (a *ZipArchiver) ArchiveMultiple(content map[string][]byte) error {
sort.Strings(keys)

for _, filename := range keys {
f, err := a.writer.Create(filepath.ToSlash(filename))
if err != nil {
return err
var f io.Writer
var err error

if normalizeFilesMetadata {
fh := prepareEmptyHeader(content[filename], filename)
normalizeCompressingFile(fh)

f, err = a.writer.CreateHeader(fh)
if err != nil {
return fmt.Errorf("error creating file inside archive: %s", err)
}
} else {
f, err = a.writer.Create(filepath.ToSlash(filename))
if err != nil {
return err
}
}

_, err = f.Write(content[filename])
if err != nil {
return err
Expand Down
Loading