Implementing path-segment sanitization rules

.circleci/config.yml

@@ -1,7 +1,7 @@
 defaults: &defaults
   working_directory: /go/src/github.com/gohugoio
   docker:
-      - image: bepsays/ci-goreleaser:0.34.2-5
+      - image: bepsays/ci-goreleaser:0.34.2-6
 
 version: 2
 jobs:

.travis.yml

@@ -2,8 +2,8 @@ language: go
 sudo: required
 dist: precise
 go:
-  - 1.8.4
-  - 1.9.1
+  - 1.8.5
+  - 1.9.2
   - tip
 os:
   - linux

Gopkg.toml

@@ -2,7 +2,7 @@
 
 [[constraint]]
   name = "github.com/BurntSushi/toml"
-  version = "0.3.0"
+  branch = "master"
 
 [[constraint]]
   name = "github.com/PuerkitoBio/purell"
@@ -24,6 +24,10 @@
   branch = "master"
   name = "github.com/dchest/cssmin"
 
+[[constraint]]
+  name = "github.com/magefile/mage"
+  version = "v1"
+
 [[constraint]]
   branch = "master"
   name = "github.com/eknkc/amber"
@@ -70,7 +74,7 @@
 
 [[constraint]]
   name = "github.com/nicksnyder/go-i18n"
-  version = "1.9.0"
+  version = "1.10.0"
 
 [[constraint]]
   name = "github.com/russross/blackfriday"

README.md

@@ -99,6 +99,7 @@ When reporting the issue, please provide the version of Hugo in use (`hugo versi
 The Hugo project welcomes all contributors and contributions regardless of skill or experience level.
 If you are interested in helping with the project, we will help you with your contribution.
 Hugo is a very active project with many contributions happening daily.
+
 Because we want to create the best possible product for our users and the best contribution experience for our developers,
 we have a set of guidelines which ensure that all contributions are acceptable.
 The guidelines are not intended as a filter or barrier to participation.

commands/hugo.go

@@ -888,7 +888,7 @@ func (c *commandeer) newWatcher(port int) error {
 					}
 
 					c.Logger.FEEDBACK.Println("\nStatic file changes detected")
-					const layout = "2006-01-02 15:04 -0700"
+					const layout = "2006-01-02 15:04:05.000 -0700"
 					c.Logger.FEEDBACK.Println(time.Now().Format(layout))
 
 					if c.Cfg.GetBool("forceSyncStatic") {

docs/content/content-management/shortcodes.md

@@ -88,6 +88,7 @@ The `figure` shortcode can use the following named parameters:
 * `attr` (i.e., attribution)
 * `attrlink`
 * `alt`
+* `height`
 
 #### Example `figure` Input
 

docs/content/content-management/taxonomies.md

@@ -132,7 +132,7 @@ Note that if you use `preserveTaxonomyNames` and intend to manually construct UR
 {{% note %}}
 You can add content and front matter to your taxonomy list and taxonomy terms pages. See [Content Organization](/content-management/organization/) for more information on how to add an `_index.md` for this purpose.
 
-Note also that taxonomy [permalinks](/content-management/urls/) are *not* configurable.
+Much like regular pages, taxonomy list [permalinks](/content-management/urls/) are configurable, but taxonomy term page permalinks are not.
 {{% /note %}}
 
 ## Add Taxonomies to Content

docs/content/content-management/urls.md

@@ -45,6 +45,8 @@ permalinks:
 
 Only the content under `post/` will have the new URL structure. For example, the file `content/post/sample-entry.md` with `date: 2017-02-27T19:20:00-05:00` in its front matter will render to `public/2017/02/sample-entry/index.html` at build time and therefore be reachable at `https://example.com/2013/11/sample-entry/`.
 
+You can also configure permalinks of taxonomies with the same syntax, by using the plural form of the taxonomy instead of the section. You will probably only want to use the configuration values `:slug` or `:title`.
+
 ### Permalink Configuration Values
 
 The following is a list of values that can be used in a `permalink` definition in your site `config` file. All references to time are dependent on the content's date.

docs/content/functions/findRe.md

@@ -45,3 +45,4 @@ If you are just learning RegEx, or at least Golang's flavor, you can practice pa
 [`plainify`]: /functions/plainify/
 [toc]: /content-management/toc/
 [`urlize`]: /functions/urlize
+[`urlizeSegment`]: /functions/urlizeSegment

docs/content/functions/urlize.md

@@ -15,7 +15,7 @@ signature: ["urlize INPUT"]
 hugoversion:
 deprecated: false
 workson: []
-relatedfuncs: []
+relatedfuncs: [urlizeSegment]
 ---
 
 The following examples pull from a content file with the following front matter:

docs/content/functions/urlizeSegment.md

@@ -0,0 +1,76 @@
+---
+title: urlizeSegment
+# linktitle: urlizeSegment
+description: Takes a string, sanitizes it for usage in URLs, and converts spaces, slashes, and pound signs to hyphens.
+date: 2017-10-21
+publishdate: 2017-10-21
+lastmod: 2017-10-21
+categories: [functions]
+menu:
+  docs:
+    parent: "functions"
+keywords: [urls,strings]
+godocref:
+signature: ["urlizeSegment INPUT"]
+hugoversion:
+deprecated: false
+workson: []
+relatedfuncs: [urlize]
+---
+
+The following examples pull from a content file with the following front matter:
+
+{{< code file="content/blog/greatest-city.md" copy="false">}}
++++
+title = "The World's Greatest City"
+location = "Chicago IL/USA"
+tags = ["food/pizza","drink/beer","food/hot dogs","we're #1"]
++++
+{{< /code >}}
+
+The following might be used as a partial within a [single page template][singletemplate]:
+
+{{< code file="layouts/partials/content-header.html" download="content-header.html" >}}
+<header>
+    <h1>{{.Title}}</h1>
+    {{ with .Params.location }}
+        <div><a href="/locations/{{ . | urlizeSegment}}">{{.}}</a></div>
+    {{ end }}
+    <!-- Creates a list of tags for the content and links to each of their pages -->
+    {{ with .Params.tags }}
+    <ul>
+        {{range .}}
+            <li>
+                <a href="/tags/{{ . | urlizeSegment }}">{{ . }}</a>
+            </li>
+        {{end}}
+    </ul>
+    {{ end }}
+</header>
+{{< /code >}}
+
+The preceding partial would then output to the rendered page as follows, assuming the page is being built with Hugo's default pretty URLs.
+
+{{< output file="/blog/greatest-city/index.html" >}}
+<header>
+    <h1>The World's Greatest City</h1>
+    <div><a href="/locations/chicago-il-usa/">Chicago IL/USA</a></div>
+    <ul>
+        <li>
+            <a href="/tags/food-pizza">food/pizza</a>
+        </li>
+        <li>
+            <a href="/tags/drink-beer">drink/beer</a>
+        </li>
+        <li>
+            <a href="/tags/food-hot-dogs">food/hot dogs</a>
+        </li>
+        <li>
+            <a href="/tags/were--1">we're #1</a>
+        </li>
+    </ul>
+</header>
+{{< /output >}}
+
+
+[singletemplate]: /templates/single-page-templates/

helpers/path.go

@@ -34,6 +34,9 @@ var (
 
 	// ErrWalkRootTooShort is returned when the root specified for a file walk is shorter than 4 characters.
 	ErrWalkRootTooShort = errors.New("Path too short. Stop walking.")
+
+	// This replacer is used in calls to MakePathSegmentSanitized.
+	SegmentReplacer = strings.NewReplacer("/", "-", "#", "-")
 )
 
 // filepathPathBridge is a bridge for common functionality in filepath vs path
@@ -92,6 +95,12 @@ func (p *PathSpec) MakePathSanitized(s string) string {
 	return strings.ToLower(p.MakePath(s))
 }
 
+// MakePathSegmentSanitized, in addition to MakePathSanitized,
+// replaces slashes and pound signs with hyphens.
+func (p *PathSpec) MakePathSegmentSanitized(s string) string {
+	return p.MakePathSanitized(SegmentReplacer.Replace(s))
+}
+
 // MakeTitle converts the path given to a suitable title, trimming whitespace
 // and replacing hyphens with whitespace.
 func MakeTitle(inpath string) string {

helpers/path_test.go

@@ -81,6 +81,7 @@ func TestMakePathSanitized(t *testing.T) {
 		{"Foo.Bar/fOO_bAr-Foo", "foo.bar/foo_bar-foo"},
 		{"FOO,bar:FooBar", "foobarfoobar"},
 		{"foo/BAR.HTML", "foo/bar.html"},
+		{"foo#BAR", "foo#bar"},
 		{"трям/трям", "трям/трям"},
 		{"은행", "은행"},
 	}
@@ -93,6 +94,32 @@ func TestMakePathSanitized(t *testing.T) {
 	}
 }
 
+func TestMakePathSegmentSanitized(t *testing.T) {
+	v := viper.New()
+	l := NewDefaultLanguage(v)
+	p, _ := NewPathSpec(hugofs.NewMem(v), l)
+
+	tests := []struct {
+		input    string
+		expected string
+	}{
+		{"  FOO bar  ", "foo-bar"},
+		{"Foo.Bar/fOO_bAr-Foo", "foo.bar-foo_bar-foo"},
+		{"FOO,bar:FooBar", "foobarfoobar"},
+		{"foo/BAR.HTML", "foo-bar.html"},
+		{"foo#BAR", "foo-bar"},
+		{"трям/трям", "трям-трям"},
+		{"은행", "은행"},
+	}
+
+	for _, test := range tests {
+		output := p.MakePathSegmentSanitized(test.input)
+		if output != test.expected {
+			t.Errorf("Expected %#v, got %#v\n", test.expected, output)
+		}
+	}
+}
+
 func TestMakePathSanitizedDisablePathToLower(t *testing.T) {
 	v := viper.New()
 

helpers/url.go

@@ -103,7 +103,14 @@ func SanitizeURLKeepTrailingSlash(in string) string {
 //     urlize: vim-text-editor
 func (p *PathSpec) URLize(uri string) string {
 	return p.URLEscape(p.MakePathSanitized(uri))
+}
 
+// URLizeSegment is URLize + slashes and pound signs become hyphens
+// Example:
+//     uri: Vi/Vim (text editor)
+//     urlizeSegment: vi-vim-text-editor
+func (p *PathSpec) URLizeSegment(uri string) string {
+	return p.URLEscape(p.MakePathSegmentSanitized(uri))
 }
 
 // URLizeFilename creates an URL from a filename by esacaping unicode letters

helpers/url_test.go

@@ -38,6 +38,7 @@ func TestURLize(t *testing.T) {
 		{"foo.bar/foo_bar-foo", "foo.bar/foo_bar-foo"},
 		{"foo,bar:foobar", "foobarfoobar"},
 		{"foo/bar.html", "foo/bar.html"},
+		{"foo#bar", "foo#bar"},
 		{"трям/трям", "%D1%82%D1%80%D1%8F%D0%BC/%D1%82%D1%80%D1%8F%D0%BC"},
 		{"100%-google", "100-google"},
 	}
@@ -50,6 +51,33 @@ func TestURLize(t *testing.T) {
 	}
 }
 
+func TestURLizeSegment(t *testing.T) {
+
+	v := viper.New()
+	l := NewDefaultLanguage(v)
+	p, _ := NewPathSpec(hugofs.NewMem(v), l)
+
+	tests := []struct {
+		input    string
+		expected string
+	}{
+		{"  foo bar  ", "foo-bar"},
+		{"foo.bar/foo_bar-foo", "foo.bar-foo_bar-foo"},
+		{"foo,bar:foobar", "foobarfoobar"},
+		{"foo/bar.html", "foo-bar.html"},
+		{"foo#bar", "foo-bar"},
+		{"трям/трям", "%D1%82%D1%80%D1%8F%D0%BC-%D1%82%D1%80%D1%8F%D0%BC"},
+		{"100%-google", "100-google"},
+	}
+
+	for _, test := range tests {
+		output := p.URLizeSegment(test.input)
+		if output != test.expected {
+			t.Errorf("Expected %#v, got %#v\n", test.expected, output)
+		}
+	}
+}
+
 func TestAbsURL(t *testing.T) {
 	for _, defaultInSubDir := range []bool{true, false} {
 		for _, addLanguage := range []bool{true, false} {

hugolib/hugo_sites.go

@@ -395,7 +395,7 @@ func (h *HugoSites) createMissingPages() error {
 						origKey := key
 
 						if s.Info.preserveTaxonomyNames {
-							key = s.PathSpec.MakePathSanitized(key)
+							key = s.PathSpec.MakePathSegmentSanitized(key)
 						}
 						for _, p := range taxonomyPages {
 							if p.sections[0] == plural && p.sections[1] == key {

hugolib/page.go

@@ -758,7 +758,7 @@ func (p *Page) Type() string {
 // since Hugo 0.22 we support nested sections, but this will always be the first
 // element of any nested path.
 func (p *Page) Section() string {
-	if p.Kind == KindSection {
+	if p.Kind == KindSection || p.Kind == KindTaxonomy || p.Kind == KindTaxonomyTerm {
 		return p.sections[0]
 	}
 	return p.Source.Section()

hugolib/pageGroup.go

@@ -142,9 +142,10 @@ func (p Pages) GroupBy(key string, order ...string) (PagesGroup, error) {
 		tmp.SetMapIndex(fv, reflect.Append(tmp.MapIndex(fv), ppv))
 	}
 
-	var r []PageGroup
-	for _, k := range sortKeys(tmp.MapKeys(), direction) {
-		r = append(r, PageGroup{Key: k.Interface(), Pages: tmp.MapIndex(k).Interface().([]*Page)})
+	sortedKeys := sortKeys(tmp.MapKeys(), direction)
+	r := make([]PageGroup, len(sortedKeys))
+	for i, k := range sortedKeys {
+		r[i] = PageGroup{Key: k.Interface(), Pages: tmp.MapIndex(k).Interface().([]*Page)}
 	}
 
 	return r, nil