Implement Page bundling and image handling

Fixes gohugoio#3651
Closes gohugoio#3158
Fixes gohugoio#1014

commands/convert.go

@@ -14,14 +14,7 @@
 package commands
 
 import (
-	"errors"
-	"fmt"
-	"path/filepath"
-	"time"
-
-	"github.com/gohugoio/hugo/hugolib"
 	"github.com/gohugoio/hugo/parser"
-	"github.com/spf13/cast"
 	"github.com/spf13/cobra"
 )
 
@@ -77,82 +70,86 @@ func init() {
 	convertCmd.PersistentFlags().SetAnnotation("source", cobra.BashCompSubdirsInDir, []string{})
 }
 
+// TODO(bep) bundles
 func convertContents(mark rune) error {
-	cfg, err := InitializeConfig()
-	if err != nil {
-		return err
-	}
-
-	h, err := hugolib.NewHugoSites(*cfg)
-	if err != nil {
-		return err
-	}
-
-	site := h.Sites[0]
-
-	if err = site.Initialise(); err != nil {
-		return err
-	}
-
-	if site.Source == nil {
-		panic("site.Source not set")
-	}
-	if len(site.Source.Files()) < 1 {
-		return errors.New("No source files found")
-	}
-
-	contentDir := site.PathSpec.AbsPathify(site.Cfg.GetString("contentDir"))
-	site.Log.FEEDBACK.Println("processing", len(site.Source.Files()), "content files")
-	for _, file := range site.Source.Files() {
-		site.Log.INFO.Println("Attempting to convert", file.LogicalName())
-		page, err := site.NewPage(file.LogicalName())
+	return nil
+	/*
+		cfg, err := InitializeConfig()
 		if err != nil {
 			return err
 		}
 
-		psr, err := parser.ReadFrom(file.Contents)
+		h, err := hugolib.NewHugoSites(*cfg)
 		if err != nil {
-			site.Log.ERROR.Println("Error processing file:", file.Path())
 			return err
 		}
-		metadata, err := psr.Metadata()
-		if err != nil {
-			site.Log.ERROR.Println("Error processing file:", file.Path())
+
+		site := h.Sites[0]
+
+		if err = site.Initialise(); err != nil {
 			return err
 		}
 
-		// better handling of dates in formats that don't have support for them
-		if mark == parser.FormatToLeadRune("json") || mark == parser.FormatToLeadRune("yaml") || mark == parser.FormatToLeadRune("toml") {
-			newMetadata := cast.ToStringMap(metadata)
-			for k, v := range newMetadata {
-				switch vv := v.(type) {
-				case time.Time:
-					newMetadata[k] = vv.Format(time.RFC3339)
-				}
-			}
-			metadata = newMetadata
+		if site.Source == nil {
+			panic("site.Source not set")
 		}
-
-		page.SetDir(filepath.Join(contentDir, file.Dir()))
-		page.SetSourceContent(psr.Content())
-		if err = page.SetSourceMetaData(metadata, mark); err != nil {
-			site.Log.ERROR.Printf("Failed to set source metadata for file %q: %s. For more info see For more info see https://github.com/gohugoio/hugo/issues/2458", page.FullFilePath(), err)
-			continue
+		if len(site.Source.Files()) < 1 {
+			return errors.New("No source files found")
 		}
 
-		if outputDir != "" {
-			if err = page.SaveSourceAs(filepath.Join(outputDir, page.FullFilePath())); err != nil {
-				return fmt.Errorf("Failed to save file %q: %s", page.FullFilePath(), err)
+		//	contentDir := site.PathSpec.AbsPathify(site.Cfg.GetString("contentDir"))
+		site.Log.FEEDBACK.Println("processing", len(site.Source.Files()), "content files")
+		for _, file := range site.Source.Files() {
+			site.Log.INFO.Println("Attempting to convert", file.LogicalName())
+			page, err := site.NewPage(file.LogicalName())
+			if err != nil {
+				return err
+			}
+
+			psr, err := parser.ReadFrom(file.Contents)
+			if err != nil {
+				site.Log.ERROR.Println("Error processing file:", file.Path())
+				return err
 			}
-		} else {
-			if unsafe {
-				if err = page.SaveSource(); err != nil {
+			metadata, err := psr.Metadata()
+			if err != nil {
+				site.Log.ERROR.Println("Error processing file:", file.Path())
+				return err
+			}
+
+			// better handling of dates in formats that don't have support for them
+			if mark == parser.FormatToLeadRune("json") || mark == parser.FormatToLeadRune("yaml") || mark == parser.FormatToLeadRune("toml") {
+				newMetadata := cast.ToStringMap(metadata)
+				for k, v := range newMetadata {
+					switch vv := v.(type) {
+					case time.Time:
+						newMetadata[k] = vv.Format(time.RFC3339)
+					}
+				}
+				metadata = newMetadata
+			}
+
+			// TODO(bep) bundles	page.SetDir(filepath.Join(contentDir, file.Dir()))
+			page.SetSourceContent(psr.Content())
+			if err = page.SetSourceMetaData(metadata, mark); err != nil {
+				site.Log.ERROR.Printf("Failed to set source metadata for file %q: %s. For more info see For more info see https://github.com/gohugoio/hugo/issues/2458", page.FullFilePath(), err)
+				continue
+			}
+
+			if outputDir != "" {
+				if err = page.SaveSourceAs(filepath.Join(outputDir, page.FullFilePath())); err != nil {
 					return fmt.Errorf("Failed to save file %q: %s", page.FullFilePath(), err)
 				}
 			} else {
-				site.Log.FEEDBACK.Println("Unsafe operation not allowed, use --unsafe or set a different output path")
+				if unsafe {
+					if err = page.SaveSource(); err != nil {
+						return fmt.Errorf("Failed to save file %q: %s", page.FullFilePath(), err)
+					}
+				} else {
+					site.Log.FEEDBACK.Println("Unsafe operation not allowed, use --unsafe or set a different output path")
+				}
 			}
 		}
-	}
-	return nil
+		return nil
+	*/
 }

commands/import_jekyll.go

@@ -468,7 +468,7 @@ func convertJekyllPost(s *hugolib.Site, path, relPath, targetDir string, draft b
 		return err
 	}
 
-	page.SetDir(targetParentDir)
+	// TODO(bep) bundlespage.SetDir(targetParentDir)
 	page.SetSourceContent([]byte(content))
 	page.SetSourceMetaData(newmetadata, parser.FormatToLeadRune("yaml"))
 	page.SaveSourceAs(targetFile)

commands/new.go

@@ -375,7 +375,11 @@ func newContentPathSection(path string) (string, string) {
 	var section string
 	// assume the first directory is the section (kind)
 	if strings.Contains(createpath[1:], helpers.FilePathSeparator) {
-		section = helpers.GuessSection(createpath)
+		parts := strings.Split(strings.TrimPrefix(createpath, helpers.FilePathSeparator), helpers.FilePathSeparator)
+		if len(parts) > 0 {
+			section = parts[0]
+		}
+
 	}
 
 	return createpath, section

create/content_template_handler.go

@@ -45,7 +45,7 @@ type ArchetypeFileData struct {
 
 	// The target content file. Note that the .Content will be empty, as that
 	// has not been created yet.
-	*source.File
+	source.File
 }
 
 const (
@@ -82,7 +82,8 @@ func executeArcheTypeAsTemplate(s *hugolib.Site, kind, targetPath, archetypeFile
 	)
 
 	sp := source.NewSourceSpec(s.Deps.Cfg, s.Deps.Fs)
-	f := sp.NewFile(targetPath)
+	// TODO(bep) bundles test
+	f := sp.NewFileInfo("", targetPath, nil)
 
 	data := ArchetypeFileData{
 		Type: kind,

deps/deps.go

@@ -10,6 +10,8 @@ import (
 	"github.com/gohugoio/hugo/hugofs"
 	"github.com/gohugoio/hugo/metrics"
 	"github.com/gohugoio/hugo/output"
+	"github.com/gohugoio/hugo/resource"
+	"github.com/gohugoio/hugo/source"
 	"github.com/gohugoio/hugo/tpl"
 	jww "github.com/spf13/jwalterweatherman"
 )
@@ -33,6 +35,13 @@ type Deps struct {
 	// The ContentSpec to use
 	*helpers.ContentSpec `json:"-"`
 
+	// The ResourceSpec to use
+	ResourceSpec *resource.Spec `json:"-"`
+
+	// The ResourceSpec to use
+	// TODO(bep) bundles remove?
+	SourceSpec *source.SourceSpec `json:"-"`
+
 	// The configuration to use
 	Cfg config.Provider `json:"-"`
 
@@ -117,19 +126,25 @@ func New(cfg DepsCfg) (*Deps, error) {
 		return nil, err
 	}
 
+	rs := resource.NewSpec(ps)
+
 	contentSpec, err := helpers.NewContentSpec(cfg.Language)
 	if err != nil {
 		return nil, err
 	}
 
+	sp := source.NewSourceSpec(cfg.Language, fs)
+
 	d := &Deps{
 		Fs:                  fs,
 		Log:                 logger,
 		templateProvider:    cfg.TemplateProvider,
 		translationProvider: cfg.TranslationProvider,
 		WithTemplate:        cfg.WithTemplate,
 		PathSpec:            ps,
+		ResourceSpec:        rs,
 		ContentSpec:         contentSpec,
+		SourceSpec:          sp,
 		Cfg:                 cfg.Language,
 		Language:            cfg.Language,
 	}

helpers/content.go

@@ -47,6 +47,10 @@ type ContentSpec struct {
 	// SummaryLength is the length of the summary that Hugo extracts from a content.
 	summaryLength int
 
+	BuildFuture  bool
+	BuildExpired bool
+	BuildDrafts  bool
+
 	Highlight            func(code, lang, optsStr string) (string, error)
 	defatultPygmentsOpts map[string]string
 
@@ -62,6 +66,9 @@ func NewContentSpec(cfg config.Provider) (*ContentSpec, error) {
 		footnoteAnchorPrefix:       cfg.GetString("footnoteAnchorPrefix"),
 		footnoteReturnLinkContents: cfg.GetString("footnoteReturnLinkContents"),
 		summaryLength:              cfg.GetInt("summaryLength"),
+		BuildFuture:                cfg.GetBool("buildFuture"),
+		BuildExpired:               cfg.GetBool("buildExpired"),
+		BuildDrafts:                cfg.GetBool("buildDrafts"),
 
 		cfg: cfg,
 	}

helpers/content_test.go

@@ -19,9 +19,12 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/spf13/viper"
+
 	"github.com/miekg/mmark"
 	"github.com/russross/blackfriday"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 const tstHTMLContent = "<!DOCTYPE html><html><head><script src=\"http://two/foobar.js\"></script></head><body><nav><ul><li hugo-nav=\"section_0\"></li><li hugo-nav=\"section_1\"></li></ul></nav><article>content <a href=\"http://two/foobar\">foobar</a>. Follow up</article><p>This is some text.<br>And some more.</p></body></html>"
@@ -73,6 +76,25 @@ func TestBytesToHTML(t *testing.T) {
 	assert.Equal(t, template.HTML("dobedobedo"), BytesToHTML([]byte("dobedobedo")))
 }
 
+func TestNewContentSpec(t *testing.T) {
+	cfg := viper.New()
+	assert := require.New(t)
+
+	cfg.Set("summaryLength", 32)
+	cfg.Set("buildFuture", true)
+	cfg.Set("buildExpired", true)
+	cfg.Set("buildDrafts", true)
+
+	spec, err := NewContentSpec(cfg)
+
+	assert.NoError(err)
+	assert.Equal(32, spec.summaryLength)
+	assert.True(spec.BuildFuture)
+	assert.True(spec.BuildExpired)
+	assert.True(spec.BuildDrafts)
+
+}
+
 var benchmarkTruncateString = strings.Repeat("This is a sentence about nothing.", 20)
 
 func BenchmarkTestTruncateWordsToWholeSentence(b *testing.B) {

helpers/path.go

@@ -277,6 +277,12 @@ func Ext(in string) string {
 	return ext
 }
 
+// FileAndExt takes a path and returns the file and extension separated,
+// the extension including the delmiter, i.e. ".md".
+func FileAndExt(in string) (string, string) {
+	return fileAndExt(in, fpb)
+}
+
 // Filename takes a path, strips out the extension,
 // and returns the name of the file.
 func Filename(in string) (name string) {
@@ -348,40 +354,6 @@ func GetRelativePath(path, base string) (final string, err error) {
 	return name, nil
 }
 
-// GuessSection returns the section given a source path.
-// A section is the part between the root slash and the second slash
-// or before the first slash.
-func GuessSection(in string) string {
-	parts := strings.Split(in, FilePathSeparator)
-	// This will include an empty entry before and after paths with leading and trailing slashes
-	// eg... /sect/one/ -> ["", "sect", "one", ""]
-
-	// Needs to have at least a value and a slash
-	if len(parts) < 2 {
-		return ""
-	}
-
-	// If it doesn't have a leading slash and value and file or trailing slash, then return ""
-	if parts[0] == "" && len(parts) < 3 {
-		return ""
-	}
-
-	// strip leading slash
-	if parts[0] == "" {
-		parts = parts[1:]
-	}
-
-	// if first directory is "content", return second directory
-	if parts[0] == "content" {
-		if len(parts) > 2 {
-			return parts[1]
-		}
-		return ""
-	}
-
-	return parts[0]
-}
-
 // PathPrep prepares the path using the uglify setting to create paths on
 // either the form /section/name/index.html or /section/name.html.
 func PathPrep(ugly bool, in string) string {