From 5ca554fdf04844991f67f9c98ecbabea582ec9f2 Mon Sep 17 00:00:00 2001 From: FuXiaoHei Date: Fri, 6 May 2022 16:36:05 +0800 Subject: [PATCH] feat: build command support --archive to create an archive with built files --- go.mod | 11 +++++++ go.sum | 26 +++++++++++++++ pkg/cmd/build.go | 19 ++++------- pkg/core/generator/context.go | 20 +++++++++--- pkg/core/generator/generate.go | 3 +- pkg/core/generator/option.go | 1 + pkg/core/generator/output.go | 59 +++++++++++++++++++++++++++++++--- pkg/core/models/output.go | 1 + 8 files changed, 119 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index 99d672f..c71f2d6 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.18 require ( github.com/BurntSushi/toml v1.1.0 github.com/fsnotify/fsnotify v1.5.4 + github.com/mholt/archiver/v4 v4.0.0-alpha.6 github.com/tdewolff/minify/v2 v2.11.1 github.com/urfave/cli/v2 v2.4.0 github.com/yuin/goldmark v1.4.11 @@ -14,9 +15,19 @@ require ( ) require ( + github.com/andybalholm/brotli v1.0.4 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect + github.com/dsnet/compress v0.0.1 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/klauspost/compress v1.15.1 // indirect + github.com/klauspost/pgzip v1.2.5 // indirect + github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect + github.com/pierrec/lz4/v4 v4.1.14 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/tdewolff/parse/v2 v2.5.29 // indirect + github.com/therootcompany/xz v1.0.1 // indirect + github.com/ulikunitz/xz v0.5.10 // indirect go.uber.org/multierr v1.6.0 // indirect golang.org/x/sys v0.0.0-20220412211240-33da011f77ad // indirect + golang.org/x/text v0.3.7 // indirect ) diff --git a/go.sum b/go.sum index 8949fc9..ed20c4f 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I= github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= +github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= @@ -10,16 +12,33 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/djherbis/atime v1.1.0/go.mod h1:28OF6Y8s3NQWwacXc5eZTsEsiMzp7LF8MbXE+XJPdBE= +github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= +github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= +github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A= +github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= +github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/matryer/try v0.0.0-20161228173917-9ac251b645a2/go.mod h1:0KeJpeMD6o+O4hW7qJOT7vyQPKrWmj26uf5wMc/IiIs= +github.com/mholt/archiver/v4 v4.0.0-alpha.6 h1:3wvos9Kn1GpKNBz+MpozinGREPslLo1ds1W16vTkErQ= +github.com/mholt/archiver/v4 v4.0.0-alpha.6/go.mod h1:9PTygYq90FQBWPspdwAng6dNjYiBuTYKqmA6c15KuCo= +github.com/nwaples/rardecode/v2 v2.0.0-beta.2 h1:e3mzJFJs4k83GXBEiTaQ5HgSc/kOK8q0rDaRO0MPaOk= +github.com/nwaples/rardecode/v2 v2.0.0-beta.2/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY= +github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE= +github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -38,6 +57,11 @@ github.com/tdewolff/parse/v2 v2.5.29 h1:Uf0OtZL9YaUXTuHEOitdo9lD90P0XTwCjZi+KbGC github.com/tdewolff/parse/v2 v2.5.29/go.mod h1:WzaJpRSbwq++EIQHYIRTpbYKNA3gn9it1Ik++q4zyho= github.com/tdewolff/test v1.0.6 h1:76mzYJQ83Op284kMT+63iCNCI7NEERsIN8dLM+RiKr4= github.com/tdewolff/test v1.0.6/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE= +github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= +github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= +github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8= +github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= +github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/urfave/cli/v2 v2.4.0 h1:m2pxjjDFgDxSPtO8WSdbndj17Wu2y8vOT86wE/tjr+I= github.com/urfave/cli/v2 v2.4.0/go.mod h1:NX9W0zmTvedE5oDoOMs2RTC8RvdK98NTYZE5LbaEYPg= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= @@ -72,6 +96,8 @@ golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= diff --git a/pkg/cmd/build.go b/pkg/cmd/build.go index 2d551c7..18d3998 100644 --- a/pkg/cmd/build.go +++ b/pkg/cmd/build.go @@ -20,6 +20,10 @@ var ( Name: "watch", Usage: "watch source files and rebuild when changed", }, + &cli.BoolFlag{ + Name: "archive", + Usage: "compress built files to one archive", + }, } ) @@ -49,18 +53,9 @@ func parseCliOption(c *cli.Context) *generator.Option { configFileItem := loadLocalConfigFile() var option = generator.Option{ ConfigFileItem: &configFileItem, + EnableWatch: c.Bool("watch"), + OutputDir: c.String("output"), + BuildArchive: c.Bool("archive"), } - - // parse output directory - if c.String("output") != "" { - option.OutputDir = c.String("output") - } - - // enable watching source files - if c.Bool("watch") { - option.EnableWatch = true - - } - return &option } diff --git a/pkg/core/generator/context.go b/pkg/core/generator/context.go index 6b25560..6d172c7 100644 --- a/pkg/core/generator/context.go +++ b/pkg/core/generator/context.go @@ -24,6 +24,8 @@ type Context struct { postSlugTemplate *template.Template tagLinkTemplate *template.Template + + allLinkFiles sync.Map } func NewContext(s *SiteData, opt *Option) *Context { @@ -124,6 +126,15 @@ func (ctx *Context) GetOutputs() []*models.OutputFile { return result } +func (ctx *Context) GetRecordFiles() []*models.OutputFile { + var result []*models.OutputFile + ctx.allLinkFiles.Range(func(key, value interface{}) bool { + result = append(result, &models.OutputFile{Link: key.(string), Path: value.(string)}) + return true + }) + return result +} + func (ctx *Context) createPostLink(p *models.Post) (string, string, error) { slugData := map[string]interface{}{ "Slug": p.Slug, @@ -156,10 +167,11 @@ func (ctx *Context) getOutputCounter() int64 { return ctx.outputCounter.Load() } -func (ctx *Context) incrOutputCounter(delta int64) int64 { - return ctx.outputCounter.Add(delta) -} - func (ctx *Context) appendCopyDir(srcDir, dstDir string) { ctx.copingDirs = append(ctx.copingDirs, &models.CopyDir{SrcDir: srcDir, DestDir: dstDir}) } + +func (ctx *Context) recordLinkFile(link, file string) { + ctx.allLinkFiles.Store(link, file) + ctx.outputCounter.Inc() +} diff --git a/pkg/core/generator/generate.go b/pkg/core/generator/generate.go index 8c44933..ec1229e 100644 --- a/pkg/core/generator/generate.go +++ b/pkg/core/generator/generate.go @@ -24,6 +24,7 @@ func Generate(opt *Option) error { if opt.OutputDir == "" { opt.OutputDir = siteData.BuildConfig.OutputDir } + zlog.Infof("output dir: %s", opt.OutputDir) // TODO: use a method to contains all extensions initialization ext.Reload(siteData.Config) @@ -35,7 +36,7 @@ func Generate(opt *Option) error { return err } - if err = Output(siteData, context, opt.OutputDir); err != nil { + if err = Output(siteData, context, opt); err != nil { zlog.Warnf("output failed: %v", err) return err } diff --git a/pkg/core/generator/option.go b/pkg/core/generator/option.go index 7ae544a..e69d7e9 100644 --- a/pkg/core/generator/option.go +++ b/pkg/core/generator/option.go @@ -8,4 +8,5 @@ type Option struct { OutputDir string EnableWatch bool IsLocalServer bool // if true, some template should be ignored, such as googleAnalytics + BuildArchive bool // if true, build archive } diff --git a/pkg/core/generator/output.go b/pkg/core/generator/output.go index 521cd9b..36738d3 100644 --- a/pkg/core/generator/output.go +++ b/pkg/core/generator/output.go @@ -2,16 +2,21 @@ package generator import ( "bytes" + "context" + "fmt" "os" "path/filepath" "pugo/pkg/core/theme" "pugo/pkg/ext/markdown" "pugo/pkg/utils" "pugo/pkg/utils/zlog" + "time" + + "github.com/mholt/archiver/v4" ) // Output outputs contents to destination directory. -func Output(s *SiteData, ctx *Context, outputDir string) error { +func Output(s *SiteData, ctx *Context, opt *Option) error { if err := updateThemeCopyDirs(s.Render, ctx); err != nil { zlog.Warn("theme: failed to update copy dirs", "err", err) return err @@ -19,9 +24,15 @@ func Output(s *SiteData, ctx *Context, outputDir string) error { if err := outputFiles(s, ctx); err != nil { return err } - if err := copyAssets(outputDir, ctx); err != nil { + if err := copyAssets(opt.OutputDir, ctx); err != nil { return err } + // BuildArchive generates archive files. + if opt.BuildArchive { + if err := buildArchive(ctx); err != nil { + return err + } + } return nil } @@ -61,7 +72,7 @@ func outputFiles(s *SiteData, ctx *Context) error { zlog.Warnf("output: failed to write file: %s, %s", fpath, err) continue } - ctx.incrOutputCounter(1) + ctx.recordLinkFile(fpath, fpath) } return nil } @@ -90,7 +101,7 @@ func copyAssets(outputDir string, ctx *Context) error { return err } zlog.Infof("assets copied: %s", dstPath) - ctx.incrOutputCounter(1) + ctx.recordLinkFile(dstPath, dstPath) return nil }) if err != nil { @@ -100,3 +111,43 @@ func copyAssets(outputDir string, ctx *Context) error { } return nil } + +func buildArchive(ctx *Context) error { + files := ctx.GetRecordFiles() + if len(files) == 0 { + return fmt.Errorf("no files to archive") + } + filesMap := make(map[string]string) + for _, file := range files { + filesMap[file.Path] = file.Path + } + + archiveFiles, err := archiver.FilesFromDisk(nil, filesMap) + if err != nil { + return err + } + + // create the output file we'll write to + filename := time.Now().Format("build-2006-01-02.tar.gz") + out, err := os.Create(filename) + if err != nil { + return err + } + defer out.Close() + + // we can use the CompressedArchive type to gzip a tarball + // (compression is not required; you could use Tar directly) + format := archiver.CompressedArchive{ + Compression: archiver.Gz{}, + Archival: archiver.Tar{}, + } + + // create the archive + err = format.Archive(context.Background(), out, archiveFiles) + if err != nil { + return err + } + + zlog.Infof("archive created: %s", filename) + return nil +} diff --git a/pkg/core/models/output.go b/pkg/core/models/output.go index 559cb11..a422b8f 100644 --- a/pkg/core/models/output.go +++ b/pkg/core/models/output.go @@ -5,5 +5,6 @@ import "bytes" // OutputFile represents a file to be generated. type OutputFile struct { Path string + Link string Buf *bytes.Buffer }