Skip to content

perf: improve pack performance, implement tree pruning #21

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

perf: improve pack performance, implement tree pruning #21

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions slug.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,14 +149,21 @@ func packWalkFn(root, src, dst string, tarW *tar.Writer, meta *Meta, dereference
return nil
}

if m := matchIgnoreRule(subpath, ignoreRules); m {
return nil
if m, r := matchIgnoreRule(subpath, ignoreRules); m && !(r.val == "**/*" && info.IsDir()) {
Copy link
Author

@skeggse skeggse Jul 21, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not terribly happy about having to include **/* in here, but it makes it possible to write ignorefiles like:

.git/
node_modules/
*
!*.tf
!*.json
!*.hcl

and get only the tf, json and hcl files. Without this, the * matches all directories and prevents recursion into anything that isn't explicitly allowed.

if info.IsDir() {
return filepath.SkipDir
} else {
return nil
}
}

// Catch directories so we don't end up with empty directories,
// the files are ignored correctly
if info.IsDir() {
if m := matchIgnoreRule(subpath+string(os.PathSeparator), ignoreRules); m {
if m, r := matchIgnoreRule(subpath+string(os.PathSeparator), ignoreRules); m {
if r.val != "**/*" { // Represents the `*` rule
return filepath.SkipDir
}
return nil
}
}
Expand Down
2 changes: 1 addition & 1 deletion slug_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ func TestPack(t *testing.T) {
// except for the .terraform/modules subdirectory.
for _, file := range fileList {
if strings.HasPrefix(file, ".terraform"+string(filepath.Separator)) &&
!strings.HasPrefix(file, filepath.Clean(".terraform/modules")) {
!strings.HasPrefix(file, filepath.Clean(".terraform/modules")) && file != ".terraform"+string(filepath.Separator) {
t.Fatalf("unexpected .terraform content: %s", file)
}
}
Expand Down
24 changes: 17 additions & 7 deletions terraformignore.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,22 +74,28 @@ func readRules(input io.Reader) []rule {
return rules
}

func matchIgnoreRule(path string, rules []rule) bool {
matched := false
func matchIgnoreRule(path string, rules []rule) (bool, *rule) {
var matched *rule = nil
path = filepath.FromSlash(path)
for _, rule := range rules {
for idx := range rules {
rule := &rules[idx]
Comment on lines +80 to +81
Copy link
Author

@skeggse skeggse Jul 21, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i don't know if there's an idiomatic go way to express this, but this is a pretty significant performance improvement

match, _ := rule.match(path)

if match {
matched = !rule.excluded
if rule.excluded {
return false, rule
Copy link
Author

@skeggse skeggse Jul 21, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This precedence matches git's behavior, in that git will see !-prefixed lines as overriding other inclusive rules.

}
if rule.val != "**/*" || matched == nil {
matched = rule
}
}
}

if matched {
if matched != nil {
debug(true, path, "Skipping excluded path:", path)
}

return matched
return matched != nil, matched
}

type rule struct {
Expand Down Expand Up @@ -200,9 +206,13 @@ var defaultExclusions = []rule{
excluded: false,
},
{
val: filepath.Join("**", ".terraform", "**"),
val: filepath.Join("**", ".terraform", "?**"),
Copy link
Author

@skeggse skeggse Jul 21, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In order to actually discovery .terraform/modules, we have to explicitly allow recursion into .terraform but not anything under it.

excluded: false,
},
{
val: filepath.Join("**", ".terraform", "modules"),
excluded: true,
},
{
val: filepath.Join("**", ".terraform", "modules", "**"),
excluded: true,
Expand Down
8 changes: 6 additions & 2 deletions terraformignore_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
func TestTerraformIgnore(t *testing.T) {
// path to directory without .terraformignore
p := parseIgnoreFile("testdata/external-dir")
if len(p) != 3 {
if len(p) != 4 {
t.Fatal("A directory without .terraformignore should get the default patterns")
}

Expand All @@ -22,6 +22,10 @@ func TestTerraformIgnore(t *testing.T) {
paths := []file{
{
path: ".terraform/",
match: false,
},
{
path: ".terraform/plugins",
match: true,
},
{
Expand Down Expand Up @@ -106,7 +110,7 @@ func TestTerraformIgnore(t *testing.T) {
},
}
for i, p := range paths {
match := matchIgnoreRule(p.path, ignoreRules)
match, _ := matchIgnoreRule(p.path, ignoreRules)
if match != p.match {
t.Fatalf("%s at index %d should be %t", p.path, i, p.match)
}
Expand Down