From 739d74de3c69b4c4fd2f2e1af1d49a210501ae4b Mon Sep 17 00:00:00 2001 From: thesayyn Date: Tue, 24 Oct 2023 09:13:35 -0700 Subject: [PATCH 1/5] feat: implement mutate command --- cmd/gomtree/cmd/mutate.go | 144 +++++++++++++++++++++++++++++++++++ cmd/gomtree/main.go | 1 + go.mod | 2 +- testdata/flat.relative.mtree | 20 +++++ 4 files changed, 166 insertions(+), 1 deletion(-) create mode 100644 cmd/gomtree/cmd/mutate.go create mode 100644 testdata/flat.relative.mtree diff --git a/cmd/gomtree/cmd/mutate.go b/cmd/gomtree/cmd/mutate.go new file mode 100644 index 00000000..bbcfc68a --- /dev/null +++ b/cmd/gomtree/cmd/mutate.go @@ -0,0 +1,144 @@ +package cmd + +import ( + "fmt" + "io" + "math" + "os" + "slices" + "strings" + + cli "github.com/urfave/cli/v2" + "github.com/vbatts/go-mtree" +) + +func NewMutateCommand() *cli.Command { + + return &cli.Command{ + Name: "mutate", + Usage: "mutate an mtree", + Description: `Mutate an mtree to have different shapes. +TODO: more info examples`, + Action: mutateAction, + ArgsUsage: " [path to output]", + Flags: []cli.Flag{ + &cli.StringSliceFlag{ + Name: "strip-prefix", + }, + &cli.BoolFlag{ + Name: "keep-comments", + Value: false, + }, + &cli.BoolFlag{ + Name: "keep-blank", + Value: false, + }, + &cli.StringFlag{ + Name: "output", + TakesFile: true, + }, + }, + } +} + +func mutateAction(c *cli.Context) error { + mtreePath := c.Args().Get(0) + outputPath := c.Args().Get(1) + stripPrexies := c.StringSlice("strip-prefix") + keepComments := c.Bool("keep-comments") + keepBlank := c.Bool("keep-blank") + + if mtreePath == "" { + return fmt.Errorf("mtree path is required.") + } else if outputPath == "" { + outputPath = mtreePath + } + + file, err := os.Open(mtreePath) + if err != nil { + return fmt.Errorf("opening %s: %w", mtreePath, err) + } + + spec, err := mtree.ParseSpec(file) + if err != nil { + return fmt.Errorf("parsing mtree %s: %w", mtreePath, err) + } + + dropDotDot := 0 + droppedParents := []*mtree.Entry{} + + entries := []mtree.Entry{} + +skip: + for _, entry := range spec.Entries { + + if !keepComments && entry.Type == mtree.CommentType { + continue + } + if !keepBlank && entry.Type == mtree.BlankType { + continue + } + + if entry.Parent != nil && slices.Contains(droppedParents, &entry) { + entry.Parent = nil + entry.Type = mtree.FullType + entry.Raw = "" + } + + if entry.Type == mtree.FullType || entry.Type == mtree.RelativeType { + fp, err := entry.Path() + // fmt.Println("fp", fp, entry.Name) + if err != nil { + return err + } + pathSegments := strings.Split(fp, "/") + + for _, prefix := range stripPrexies { + + prefixSegments := strings.Split(prefix, "/") + minLen := int(math.Min(float64(len(pathSegments)), float64(len(prefixSegments)))) + matches := make([]string, minLen) + for i := 0; i < minLen; i++ { + if pathSegments[i] == prefixSegments[i] { + matches[i] = prefixSegments[i] + } + } + + strip := strings.Join(matches, "/") + if entry.Type == mtree.FullType { + entry.Name = strings.TrimPrefix(entry.Name, strip) + entry.Name = strings.TrimPrefix(entry.Name, "/") + if entry.Name == "" { + continue skip + } + } else { + if entry.IsDir() { + dropDotDot++ + droppedParents = append(droppedParents, &entry) + } + if fp == strip { + continue skip + } + } + } + } else if dropDotDot > 0 && entry.Type == mtree.DotDotType { + dropDotDot-- + continue skip + } + entries = append(entries, entry) + } + + spec.Entries = entries + + var writer io.Writer = os.Stdout + if outputPath != "-" { + writer, err = os.Create(outputPath) + if err != nil { + return fmt.Errorf("creating output %s: %w", outputPath, err) + } + } + + spec.WriteTo(writer) + + return nil +} diff --git a/cmd/gomtree/main.go b/cmd/gomtree/main.go index 90fff237..3de58250 100644 --- a/cmd/gomtree/main.go +++ b/cmd/gomtree/main.go @@ -42,6 +42,7 @@ to support xattrs and interacting with tar archives.` } app.Commands = []*cli.Command{ cmd.NewValidateCommand(), + cmd.NewMutateCommand(), } // Unfortunately urfave/cli is not at good at using DefaultCommand diff --git a/go.mod b/go.mod index 1869eed0..ead060ee 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/vbatts/go-mtree -go 1.17 +go 1.18 require ( github.com/davecgh/go-spew v1.1.1 diff --git a/testdata/flat.relative.mtree b/testdata/flat.relative.mtree new file mode 100644 index 00000000..e575ec34 --- /dev/null +++ b/testdata/flat.relative.mtree @@ -0,0 +1,20 @@ +# ./lib +lib type=dir mode=0644 + foo mode=0644 size=12288 time=1457644483.833957552 type=file + +# ./lib/dir +lib type=dir mode=0644 + +dir type=dir mode=0644 + +# ./lib +.. + +# . +.. + +# ./lib/dir/sub +lib/dir/sub type=dir +lib/dir/sub/file.txt type=file + +lib/dir/PKG.info type=file mode=0644 From cc12bd7a0bb1fb0ccda82d9b571eea8be9c9ed7a Mon Sep 17 00:00:00 2001 From: thesayyn Date: Tue, 24 Oct 2023 15:58:31 -0700 Subject: [PATCH 2/5] visitor pattern --- cmd/gomtree/cmd/mutate.go | 133 +++++++++++------- entry.go | 5 +- parse.go | 1 + .../{flat.relative.mtree => relative.mtree} | 11 +- 4 files changed, 93 insertions(+), 57 deletions(-) rename testdata/{flat.relative.mtree => relative.mtree} (68%) diff --git a/cmd/gomtree/cmd/mutate.go b/cmd/gomtree/cmd/mutate.go index bbcfc68a..1114b5b6 100644 --- a/cmd/gomtree/cmd/mutate.go +++ b/cmd/gomtree/cmd/mutate.go @@ -64,67 +64,46 @@ func mutateAction(c *cli.Context) error { return fmt.Errorf("parsing mtree %s: %w", mtreePath, err) } - dropDotDot := 0 - droppedParents := []*mtree.Entry{} + stripPrefixVisitor := stripPrefixVisitor{ + prefixes: stripPrexies, + } + tidyVisitor := tidyVisitor{ + keepComments: keepComments, + keepBlank: keepBlank, + } + visitors := []Visitor{ + &stripPrefixVisitor, + &tidyVisitor, + } + dropped := []int{} entries := []mtree.Entry{} skip: for _, entry := range spec.Entries { - if !keepComments && entry.Type == mtree.CommentType { - continue - } - if !keepBlank && entry.Type == mtree.BlankType { - continue - } - - if entry.Parent != nil && slices.Contains(droppedParents, &entry) { - entry.Parent = nil - entry.Type = mtree.FullType - entry.Raw = "" + if entry.Parent != nil && slices.Contains(dropped, entry.Parent.Pos) { + if entry.Type == mtree.DotDotType { + // directory for this .. has been dropped so shall this + continue + } + entry.Parent = entry.Parent.Parent + // TODO: i am not sure if this is the correct behavior + entry.Raw = strings.TrimPrefix(entry.Raw, " ") } - if entry.Type == mtree.FullType || entry.Type == mtree.RelativeType { - fp, err := entry.Path() - // fmt.Println("fp", fp, entry.Name) + for _, visitor := range visitors { + drop, err := visitor.Visit(&entry) if err != nil { return err } - pathSegments := strings.Split(fp, "/") - - for _, prefix := range stripPrexies { - - prefixSegments := strings.Split(prefix, "/") - minLen := int(math.Min(float64(len(pathSegments)), float64(len(prefixSegments)))) - matches := make([]string, minLen) - for i := 0; i < minLen; i++ { - if pathSegments[i] == prefixSegments[i] { - matches[i] = prefixSegments[i] - } - } - - strip := strings.Join(matches, "/") - if entry.Type == mtree.FullType { - entry.Name = strings.TrimPrefix(entry.Name, strip) - entry.Name = strings.TrimPrefix(entry.Name, "/") - if entry.Name == "" { - continue skip - } - } else { - if entry.IsDir() { - dropDotDot++ - droppedParents = append(droppedParents, &entry) - } - if fp == strip { - continue skip - } - } + + if drop { + dropped = append(dropped, entry.Pos) + continue skip } - } else if dropDotDot > 0 && entry.Type == mtree.DotDotType { - dropDotDot-- - continue skip } + entries = append(entries, entry) } @@ -142,3 +121,61 @@ skip: return nil } + +type Visitor interface { + Visit(entry *mtree.Entry) (bool, error) +} + +type tidyVisitor struct { + keepComments bool + keepBlank bool +} + +func (m *tidyVisitor) Visit(entry *mtree.Entry) (bool, error) { + if !m.keepComments && entry.Type == mtree.CommentType { + return true, nil + } else if !m.keepBlank && entry.Type == mtree.BlankType { + return true, nil + } + return false, nil +} + +type stripPrefixVisitor struct { + prefixes []string +} + +func (m *stripPrefixVisitor) Visit(entry *mtree.Entry) (bool, error) { + if entry.Type != mtree.FullType && entry.Type != mtree.RelativeType { + return false, nil + } + + fp, err := entry.Path() + if err != nil { + return false, err + } + pathSegments := strings.Split(fp, "/") + + for _, prefix := range m.prefixes { + + prefixSegments := strings.Split(prefix, "/") + minLen := int(math.Min(float64(len(pathSegments)), float64(len(prefixSegments)))) + matches := make([]string, minLen) + for i := 0; i < minLen; i++ { + if pathSegments[i] == prefixSegments[i] { + matches[i] = prefixSegments[i] + } + } + + strip := strings.Join(matches, "/") + if entry.Type == mtree.FullType { + entry.Name = strings.TrimPrefix(entry.Name, strip) + entry.Name = strings.TrimPrefix(entry.Name, "/") + if entry.Name == "" { + return true, nil + } + } else if fp == strip { + return true, nil + } + } + return false, nil +} diff --git a/entry.go b/entry.go index 366a15b4..084d5956 100644 --- a/entry.go +++ b/entry.go @@ -134,7 +134,10 @@ func (e Entry) String() string { if e.Type == SpecialType || e.Type == FullType || inKeyValSlice("type=dir", e.Keywords) { return fmt.Sprintf("%s %s", e.Name, strings.Join(KeyValToString(e.Keywords), " ")) } - return fmt.Sprintf(" %s %s", e.Name, strings.Join(KeyValToString(e.Keywords), " ")) + if e.Parent != nil && e.Type != DotDotType { + return fmt.Sprintf(" %s %s", e.Name, strings.Join(KeyValToString(e.Keywords), " ")) + } + return fmt.Sprintf("%s %s", e.Name, strings.Join(KeyValToString(e.Keywords), " ")) } // AllKeys returns the full set of KeyVal for the given entry, based on the diff --git a/parse.go b/parse.go index e385eaf1..8cafba04 100644 --- a/parse.go +++ b/parse.go @@ -58,6 +58,7 @@ func ParseSpec(r io.Reader) (*DirectoryHierarchy, error) { e.Type = DotDotType e.Raw = str if creator.curDir != nil { + e.Parent = creator.curDir creator.curDir = creator.curDir.Parent } // nothing else to do here diff --git a/testdata/flat.relative.mtree b/testdata/relative.mtree similarity index 68% rename from testdata/flat.relative.mtree rename to testdata/relative.mtree index e575ec34..456e49ea 100644 --- a/testdata/flat.relative.mtree +++ b/testdata/relative.mtree @@ -2,18 +2,13 @@ lib type=dir mode=0644 foo mode=0644 size=12288 time=1457644483.833957552 type=file -# ./lib/dir -lib type=dir mode=0644 +.. -dir type=dir mode=0644 +./lib type=dir mode=0644 -# ./lib -.. -# . -.. +ayo mode=0644 size=12288 time=1457644483.833957552 type=file -# ./lib/dir/sub lib/dir/sub type=dir lib/dir/sub/file.txt type=file From d843e933dfbd0f8f8717648100abf719b714383d Mon Sep 17 00:00:00 2001 From: thesayyn Date: Wed, 1 Nov 2023 14:10:04 -0700 Subject: [PATCH 3/5] dont use slices --- cmd/gomtree/cmd/mutate.go | 22 +++++++++++----------- go.mod | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cmd/gomtree/cmd/mutate.go b/cmd/gomtree/cmd/mutate.go index 1114b5b6..7d858ce2 100644 --- a/cmd/gomtree/cmd/mutate.go +++ b/cmd/gomtree/cmd/mutate.go @@ -5,7 +5,6 @@ import ( "io" "math" "os" - "slices" "strings" cli "github.com/urfave/cli/v2" @@ -76,20 +75,21 @@ func mutateAction(c *cli.Context) error { &tidyVisitor, } - dropped := []int{} + dropped := map[int]bool{} entries := []mtree.Entry{} skip: for _, entry := range spec.Entries { - - if entry.Parent != nil && slices.Contains(dropped, entry.Parent.Pos) { - if entry.Type == mtree.DotDotType { - // directory for this .. has been dropped so shall this - continue + if entry.Parent != nil { + if _, ok := dropped[entry.Parent.Pos]; ok { + if entry.Type == mtree.DotDotType { + // directory for this .. has been dropped so shall this + continue + } + entry.Parent = entry.Parent.Parent + // TODO: i am not sure if this is the correct behavior + entry.Raw = strings.TrimPrefix(entry.Raw, " ") } - entry.Parent = entry.Parent.Parent - // TODO: i am not sure if this is the correct behavior - entry.Raw = strings.TrimPrefix(entry.Raw, " ") } for _, visitor := range visitors { @@ -99,7 +99,7 @@ skip: } if drop { - dropped = append(dropped, entry.Pos) + dropped[entry.Pos] = true continue skip } } diff --git a/go.mod b/go.mod index ead060ee..1869eed0 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/vbatts/go-mtree -go 1.18 +go 1.17 require ( github.com/davecgh/go-spew v1.1.1 From 9c8328e386591250a0f762da577ab17410997eb1 Mon Sep 17 00:00:00 2001 From: thesayyn Date: Fri, 3 Nov 2023 11:47:53 -0700 Subject: [PATCH 4/5] handle error --- cmd/gomtree/cmd/mutate.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/gomtree/cmd/mutate.go b/cmd/gomtree/cmd/mutate.go index 7d858ce2..92f35e09 100644 --- a/cmd/gomtree/cmd/mutate.go +++ b/cmd/gomtree/cmd/mutate.go @@ -117,7 +117,10 @@ skip: } } - spec.WriteTo(writer) + _, err = spec.WriteTo(writer) + if err != nil { + return fmt.Errorf("writing mtree %s: %w", outputPath, err) + } return nil } From 711c1b02596205262801116c20b19ee8a8ef5243 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Thu, 14 Mar 2024 13:40:07 -0400 Subject: [PATCH 5/5] Makefile: ensure tests are not parallel Signed-off-by: Vincent Batts --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index ac31e471..f15fdfa0 100644 --- a/Makefile +++ b/Makefile @@ -39,10 +39,10 @@ CLEAN_FILES += .test .test.tags NO_VENDOR_DIR := $(shell find . -type f -name '*.go' ! -path './vendor*' ! -path './.git*' ! -path './.vscode*' -exec dirname "{}" \; | sort -u) .test: $(SOURCE_FILES) - go test -v $(NO_VENDOR_DIR) && touch $@ + go test -p 1 -v $(NO_VENDOR_DIR) && touch $@ .test.tags: $(SOURCE_FILES) - set -e ; for tag in $(TAGS) ; do go test -tags $$tag -v $(NO_VENDOR_DIR) ; done && touch $@ + set -e ; for tag in $(TAGS) ; do go test -p 1 -tags $$tag -v $(NO_VENDOR_DIR) ; done && touch $@ .PHONY: lint lint: .lint