Support multiple content directories

This is an initial commit to fix implement this feature, it is still
missing tests. I've tested it on an example site, but I still need to
write some tests. I'm pushing it now just to ask for reviews.

In the
[forum](https://discourse.gohugo.io/t/multiple-contentdirs/7462/4) @bep
recommends to use `UnionFile`, but this solution is not using it, I am
asking for reviews for this approach, but will happily implement it
using `UnionFile` if it's better.

Fixes #3757

commands/convert.go

@@ -19,6 +19,7 @@ import (
 	"path/filepath"
 	"time"
 
+	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugolib"
 	"github.com/gohugoio/hugo/parser"
 	"github.com/spf13/cast"
@@ -101,7 +102,6 @@ func convertContents(mark rune) error {
 		return errors.New("No source files found")
 	}
 
-	contentDir := site.PathSpec.AbsPathify(site.Cfg.GetString("contentDir"))
 	site.Log.FEEDBACK.Println("processing", len(site.Source.Files()), "content files")
 	for _, file := range site.Source.Files() {
 		site.Log.INFO.Println("Attempting to convert", file.LogicalName())
@@ -133,7 +133,12 @@ func convertContents(mark rune) error {
 			metadata = newMetadata
 		}
 
-		page.SetDir(filepath.Join(contentDir, file.Dir()))
+		for _, contentDir := range helpers.GetContentDirsAbsolutePaths(site.Cfg, site.PathSpec) {
+			if contentDir == file.BasePath() {
+				page.SetDir(filepath.Join(contentDir, file.Dir()))
+				break
+			}
+		}
 		page.SetSourceContent(psr.Content())
 		if err = page.SetSourceMetaData(metadata, mark); err != nil {
 			site.Log.ERROR.Printf("Failed to set source metadata for file %q: %s. For more info see For more info see https://github.com/gohugoio/hugo/issues/2458", page.FullFilePath(), err)

commands/hugo.go

@@ -151,6 +151,7 @@ var (
 	baseURL         string
 	cacheDir        string
 	contentDir      string
+	contentDirs     []string
 	layoutDir       string
 	cfgFile         string
 	destination     string
@@ -248,6 +249,7 @@ func initHugoBuildCommonFlags(cmd *cobra.Command) {
 	cmd.Flags().BoolVarP(&logI18nWarnings, "i18n-warnings", "", false, "print missing translations")
 
 	cmd.Flags().StringSliceVar(&disableKinds, "disableKinds", []string{}, "disable different kind of pages (home, RSS etc.)")
+	cmd.Flags().StringSliceVar(&contentDirs, "contentDirs", []string{}, "filesystem paths to content directories. Mutually exclusive with contentDir")
 
 	// Set bash-completion.
 	// Each flag must first be defined before using the SetAnnotation() call.
@@ -361,6 +363,14 @@ func InitializeConfig(subCmdVs ...*cobra.Command) (*deps.DepsCfg, error) {
 		config.Set("contentDir", contentDir)
 	}
 
+	if len(contentDirs) > 0 {
+		config.Set("contentDirs", contentDirs)
+	}
+
+	if contentDir != "" && len(contentDirs) > 0 {
+		return nil, fmt.Errorf("contentDirs and contentDir are both defined. You can only define one of them.")
+	}
+
 	if layoutDir != "" {
 		config.Set("layoutDir", layoutDir)
 	}
@@ -532,7 +542,8 @@ func (c *commandeer) build(watches ...bool) error {
 	}
 
 	if buildWatch {
-		c.Logger.FEEDBACK.Println("Watching for changes in", c.PathSpec().AbsPathify(c.Cfg.GetString("contentDir")))
+		contentDirs := helpers.GetContentDirsAbsolutePaths(c.Cfg, c.PathSpec())
+		c.Logger.FEEDBACK.Printf("Watching for changes in %s\n", strings.Join(contentDirs, ", "))
 		c.Logger.FEEDBACK.Println("Press Ctrl+C to stop")
 		utils.CheckErr(c.Logger, c.newWatcher(0))
 	}
@@ -699,7 +710,9 @@ func (c *commandeer) getDirList() []string {
 
 	// SymbolicWalk will log anny ERRORs
 	_ = helpers.SymbolicWalk(c.Fs.Source, dataDir, walker)
-	_ = helpers.SymbolicWalk(c.Fs.Source, c.PathSpec().AbsPathify(c.Cfg.GetString("contentDir")), walker)
+	for _, contentDir := range helpers.GetContentDirsAbsolutePaths(c.Cfg, c.PathSpec()) {
+		_ = helpers.SymbolicWalk(c.Fs.Source, contentDir, walker)
+	}
 	_ = helpers.SymbolicWalk(c.Fs.Source, i18nDir, walker)
 	_ = helpers.SymbolicWalk(c.Fs.Source, layoutDir, walker)
 	_ = helpers.SymbolicWalk(c.Fs.Source, staticDir, walker)

create/content.go

@@ -16,16 +16,18 @@ package create
 
 import (
 	"bytes"
+	"fmt"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 
 	"github.com/gohugoio/hugo/helpers"
 	"github.com/gohugoio/hugo/hugolib"
 	jww "github.com/spf13/jwalterweatherman"
 )
 
-// NewContent creates a new content file in the content directory based upon the
+// NewContent creates a new content file in a content directory based upon the
 // given kind, which is used to lookup an archetype.
 func NewContent(
 	ps *helpers.PathSpec,
@@ -63,10 +65,23 @@ func NewContent(
 		return err
 	}
 
-	contentPath := s.PathSpec.AbsPathify(filepath.Join(s.Cfg.GetString("contentDir"), targetPath))
+	var contentPath string
+	errs := make([]string, 0)
+	contentDirs := helpers.GetContentDirs(s.Cfg)
+	// Try to create the content in content directories, keep track of errors
+	for _, contentDir := range contentDirs {
+		contentPath = s.PathSpec.AbsPathify(filepath.Join(contentDir, targetPath))
+		err := helpers.SafeWriteToDisk(contentPath, bytes.NewReader(content), s.Fs.Source)
+		if err != nil {
+			errs = append(errs, fmt.Sprintf("Directory %s - Error %s", contentDir, err))
+		} else {
+			break
+		}
+	}
 
-	if err := helpers.SafeWriteToDisk(contentPath, bytes.NewReader(content), s.Fs.Source); err != nil {
-		return err
+	// If all content directories generated errors, then content cannot be created
+	if len(errs) == len(contentDirs) {
+		return fmt.Errorf("Unable to create content in all content directories: %s.\n", strings.Join(errs, ", "))
 	}
 
 	jww.FEEDBACK.Println(contentPath, "created")

create/content_test.go

@@ -35,9 +35,6 @@ import (
 )
 
 func TestNewContent(t *testing.T) {
-	v := viper.New()
-	initViper(v)
-
 	cases := []struct {
 		kind     string
 		path     string

docs/content/commands/hugo.md

@@ -34,6 +34,7 @@ hugo [flags]
       --cleanDestinationDir        remove files from destination not found in static directories
       --config string              config file (default is path/config.yaml|json|toml)
   -c, --contentDir string          filesystem path to content directory
+      --contentDirs stringSlice    filesystem paths to content directories
   -d, --destination string         filesystem path to write files to
       --disable404                 do not render 404 page
       --disableKinds stringSlice   disable different kind of pages (home, RSS etc.)

docs/content/commands/hugo_benchmark.md

@@ -29,6 +29,7 @@ hugo benchmark [flags]
       --canonifyURLs               if true, all relative URLs will be canonicalized using baseURL
       --cleanDestinationDir        remove files from destination not found in static directories
   -c, --contentDir string          filesystem path to content directory
+      --contentDirs stringSlice    filesystem paths to content directories
   -n, --count int                  number of times to build the site (default 13)
       --cpuprofile string          path/filename for the CPU profile file
   -d, --destination string         filesystem path to write files to

docs/content/commands/hugo_server.md

@@ -41,6 +41,7 @@ hugo server [flags]
       --canonifyURLs               if true, all relative URLs will be canonicalized using baseURL
       --cleanDestinationDir        remove files from destination not found in static directories
   -c, --contentDir string          filesystem path to content directory
+      --contentDirs stringSlice    filesystem paths to content directories
   -d, --destination string         filesystem path to write files to
       --disable404                 do not render 404 page
       --disableKinds stringSlice   disable different kind of pages (home, RSS etc.)

docs/content/getting-started/configuration.md

@@ -75,6 +75,7 @@ canonifyURLs:               false
 # config file (default is path/config.yaml|json|toml)
 config:                     "config.toml"
 contentDir:                 "content"
+contentDirs:                []
 dataDir:                    "data"
 defaultExtension:           "html"
 defaultLayout:              "post"
@@ -216,6 +217,7 @@ canonifyURLs =                false
 # config file (default is path/config.yaml|json|toml)
 config =                     "config.toml"
 contentDir =                  "content"
+contentDirs =                 []
 dataDir =                     "data"
 defaultExtension =            "html"
 defaultLayout =               "post"

docs/content/getting-started/usage.md

@@ -67,6 +67,7 @@ Flags:
       --cleanDestinationDir        remove files from destination not found in static directories
       --config string              config file (default is path/config.yaml|json|toml)
   -c, --contentDir string          filesystem path to content directory
+      --contentDirs stringSlice    filesystem paths to content directories
   -d, --destination string         filesystem path to write files to
       --disable404                 do not render 404 page
       --disableKinds stringSlice   disable different kind of pages (home, RSS etc.)

helpers/contentDirs.go

@@ -0,0 +1,40 @@
+// Copyright 2017-present The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package helpers
+
+import (
+	"github.com/gohugoio/hugo/config"
+	"github.com/spf13/cast"
+)
+
+func GetContentDirsNoFallbackToContentDir(cfg config.Provider) []string {
+	contentDirs := cast.ToStringSlice(cfg.Get("contentDirs"))
+	return contentDirs
+}
+
+func GetContentDirs(cfg config.Provider) []string {
+	contentDirs := GetContentDirsNoFallbackToContentDir(cfg)
+	if len(contentDirs) == 0 && cfg.IsSet("contentDir") {
+		contentDirs = append(contentDirs, cfg.GetString("contentDir"))
+	}
+	return contentDirs
+}
+
+func GetContentDirsAbsolutePaths(cfg config.Provider, ps *PathSpec) []string {
+	contentDirs := GetContentDirs(cfg)
+	for i, contentDir := range contentDirs {
+		contentDirs[i] = ps.AbsPathify(contentDir)
+	}
+	return contentDirs
+}

hugolib/config.go

@@ -67,6 +67,12 @@ func LoadConfig(fs afero.Fs, relativeSourcePath, configFilename string) (*viper.
 		helpers.Deprecated("site config", "disableRobotsTXT", "Use disableKinds= [\"robotsTXT\"]", false)
 	}
 
+	if v.IsSet("contentDir") && v.IsSet("contentDirs") && len(helpers.GetContentDirsNoFallbackToContentDir(v)) != 0 {
+		return nil, fmt.Errorf("contentDirs and contentDir are both defined. You can only define one of them.\n")
+	} else if v.IsSet("contentDir") {
+		v.Set("contentDirs", []string{v.GetString("contentDir")})
+	}
+
 	loadDefaultSettingsFor(v)
 
 	return v, nil
@@ -84,6 +90,7 @@ func loadDefaultSettingsFor(v *viper.Viper) {
 	v.SetDefault("disableSitemap", false)
 	v.SetDefault("disableRobotsTXT", false)
 	v.SetDefault("contentDir", "content")
+	v.SetDefault("contentDirs", make([]string, 0))
 	v.SetDefault("layoutDir", "layouts")
 	v.SetDefault("staticDir", "static")
 	v.SetDefault("archetypeDir", "archetypes")

hugolib/config_test.go

@@ -16,6 +16,7 @@ package hugolib
 import (
 	"testing"
 
+	"github.com/gohugoio/hugo/helpers"
 	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -40,4 +41,35 @@ func TestLoadConfig(t *testing.T) {
 	assert.Equal(t, "side", cfg.GetString("paginatePath"))
 	// default
 	assert.Equal(t, "layouts", cfg.GetString("layoutDir"))
+
+	// contentDir checks
+	configContent = `
+	contentDir = "content"
+	contentDirs = ["content"]
+	`
+	writeToFs(t, mm, "hugo.toml", configContent)
+	cfg, err = LoadConfig(mm, "", "hugo.toml")
+	require.Error(t, err)
+
+	configContent = `
+	contentDir = "content"
+	`
+	writeToFs(t, mm, "hugo.toml", configContent)
+	cfg, err = LoadConfig(mm, "", "hugo.toml")
+	require.NoError(t, err)
+	assert.Equal(t, "content", cfg.GetString("contentDir"))
+	contentDirs := helpers.GetContentDirs(cfg)
+	assert.Equal(t, 1, len(contentDirs))
+	assert.Equal(t, "content", contentDirs[0])
+
+	configContent = `
+	contentDirs = ["content1", "content2"]
+	`
+	writeToFs(t, mm, "hugo.toml", configContent)
+	cfg, err = LoadConfig(mm, "", "hugo.toml")
+	require.NoError(t, err)
+	contentDirs = helpers.GetContentDirs(cfg)
+	assert.Equal(t, 2, len(contentDirs))
+	assert.Equal(t, "content1", contentDirs[0])
+	assert.Equal(t, "content2", contentDirs[1])
 }

hugolib/gitinfo.go

@@ -28,8 +28,8 @@ func (h *HugoSites) assembleGitInfo() {
 	}
 
 	var (
-		workingDir = h.Cfg.GetString("workingDir")
-		contentDir = h.Cfg.GetString("contentDir")
+		workingDir  = h.Cfg.GetString("workingDir")
+		contentDirs = helpers.GetContentDirs(h.Cfg)
 	)
 
 	gitRepo, err := gitmap.Map(workingDir, "")
@@ -54,16 +54,24 @@ func (h *HugoSites) assembleGitInfo() {
 			// Home page etc. with no content file.
 			continue
 		}
-		// Git normalizes file paths on this form:
-		filename := path.Join(filepath.ToSlash(contentRoot), contentDir, filepath.ToSlash(p.Path()))
-		g, ok := gitMap[filename]
-		if !ok {
-			h.Log.WARN.Printf("Failed to find GitInfo for %q", filename)
+		// Count how many failed attempts we did
+		errs := 0
+		for _, contentDir := range contentDirs {
+			// Git normalizes file paths on this form:
+			filename := path.Join(filepath.ToSlash(contentRoot), contentDir, filepath.ToSlash(p.Path()))
+			g, ok := gitMap[filename]
+			if !ok {
+				errs++
+			} else {
+				p.GitInfo = g
+				p.Lastmod = p.GitInfo.AuthorDate
+				return
+			}
+		}
+		// If we failed to find GitInfo in all contentDirs
+		if errs == len(contentDirs) {
+			h.Log.WARN.Printf("Failed to find GitInfo for %q", filepath.ToSlash(p.Path()))
 			return
 		}
-
-		p.GitInfo = g
-		p.Lastmod = p.GitInfo.AuthorDate
 	}
-
 }

hugolib/hugo_sites.go

@@ -44,21 +44,26 @@ type HugoSites struct {
 // Returns nil if none found.
 func (h *HugoSites) GetContentPage(filename string) *Page {
 	s := h.Sites[0]
-	contendDir := filepath.Join(s.PathSpec.AbsPathify(s.Cfg.GetString("contentDir")))
-	if !strings.HasPrefix(filename, contendDir) {
-		return nil
-	}
+	contentDirs := helpers.GetContentDirs(s.Cfg)
 
-	rel := strings.TrimPrefix(filename, contendDir)
-	rel = strings.TrimPrefix(rel, helpers.FilePathSeparator)
+	for _, contentDir := range contentDirs {
+		contentDir = filepath.Join(s.PathSpec.AbsPathify(contentDir))
+		if !strings.HasPrefix(filename, contentDir) {
+			continue
+		}
+		rel := strings.TrimPrefix(filename, contentDir)
+		rel = strings.TrimPrefix(rel, helpers.FilePathSeparator)
 
-	pos := s.rawAllPages.findPagePosByFilePath(rel)
+		pos := s.rawAllPages.findPagePosByFilePath(rel)
 
-	if pos == -1 {
-		return nil
+		if pos == -1 {
+			continue
+		}
+		return s.rawAllPages[pos]
 	}
-	return s.rawAllPages[pos]
 
+	// If content page is not found in all contentDirs
+	return nil
 }
 
 // NewHugoSites creates a new collection of sites given the input sites, building

hugolib/page.go

@@ -1070,10 +1070,14 @@ func (p *Page) update(f interface{}) error {
 	p.Params["draft"] = p.Draft
 
 	if p.Date.IsZero() && p.s.Cfg.GetBool("useModTimeAsFallback") {
-		fi, err := p.s.Fs.Source.Stat(filepath.Join(p.s.PathSpec.AbsPathify(p.s.Cfg.GetString("contentDir")), p.File.Path()))
-		if err == nil {
-			p.Date = fi.ModTime()
-			p.Params["date"] = p.Date
+		contentDirs := helpers.GetContentDirs(p.s.Cfg)
+		for _, contentDir := range contentDirs {
+			fi, err := p.s.Fs.Source.Stat(filepath.Join(p.s.PathSpec.AbsPathify(contentDir), p.File.Path()))
+			if err == nil {
+				p.Date = fi.ModTime()
+				p.Params["date"] = p.Date
+				break
+			}
 		}
 	}