Skip to content

Commit

Permalink
Only invoke a given cached partial once
Browse files Browse the repository at this point in the history
Note that this is backed by a LRU cache (which we soon shall see more usage of), so if you're a heavy user of cached partials it may be evicted and
refreshed if needed. But in most cases every partial is only invoked once.

This commit also adds a timeout (the global `timeout` config option) to make infinite recursion in partials
easier to reason about.

```
name              old time/op    new time/op    delta
IncludeCached-10    8.92ms ± 0%    8.48ms ± 1%   -4.87%  (p=0.016 n=4+5)

name              old alloc/op   new alloc/op   delta
IncludeCached-10    6.65MB ± 0%    5.17MB ± 0%  -22.32%  (p=0.002 n=6+6)

name              old allocs/op  new allocs/op  delta
IncludeCached-10      117k ± 0%       71k ± 0%  -39.44%  (p=0.002 n=6+6)
```

Closes gohugoio#4086
Updates gohugoio#9588
  • Loading branch information
bep committed Jan 25, 2023
1 parent 93ed6e4 commit 8219147
Show file tree
Hide file tree
Showing 18 changed files with 274 additions and 177 deletions.
4 changes: 4 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ require (
github.com/bep/godartsass v0.16.0
github.com/bep/golibsass v1.1.0
github.com/bep/gowebp v0.2.0
github.com/bep/lazycache v0.1.0
github.com/bep/overlayfs v0.6.0
github.com/bep/tmc v0.5.1
github.com/clbanning/mxj/v2 v2.5.7
Expand Down Expand Up @@ -103,6 +104,7 @@ require (
github.com/google/wire v0.5.0 // indirect
github.com/googleapis/gax-go/v2 v2.3.0 // indirect
github.com/googleapis/go-type-adapters v1.0.0 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect
github.com/inconshreveable/mousetrap v1.0.1 // indirect
github.com/invopop/yaml v0.1.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
Expand All @@ -127,3 +129,5 @@ require (
)

go 1.18

replace github.com/bep/lazycache => /Users/bep/dev/go/bep/lazycache
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -238,8 +238,6 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m
github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0=
github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/evanw/esbuild v0.15.18 h1:CM7eAoUjjNkZs1LH0p6fkwtADrbFr4JV2SlT1bUMjEo=
github.com/evanw/esbuild v0.15.18/go.mod h1:iINY06rn799hi48UqEnaQvVfZWe6W9bET78LbvN8VWk=
github.com/evanw/esbuild v0.17.0 h1:gGx9TCZDO9k9x1PJdizx6syIpUq29RwrtHWlgDIdQH8=
github.com/evanw/esbuild v0.17.0/go.mod h1:iINY06rn799hi48UqEnaQvVfZWe6W9bET78LbvN8VWk=
github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
Expand Down Expand Up @@ -401,6 +399,8 @@ github.com/hairyhenderson/go-codeowners v0.2.3-0.20201026200250-cdc7c0759690 h1:
github.com/hairyhenderson/go-codeowners v0.2.3-0.20201026200250-cdc7c0759690/go.mod h1:8Qu9UmnhCRunfRv365Z3w+mT/WfLGKJiK+vugY9qNCU=
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4=
github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
Expand Down
22 changes: 1 addition & 21 deletions helpers/general.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,13 @@ import (
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"unicode"
"unicode/utf8"

"github.com/gohugoio/hugo/common/loggers"

"github.com/mitchellh/hashstructure"

"github.com/gohugoio/hugo/common/hugo"

"github.com/spf13/afero"
Expand Down Expand Up @@ -219,7 +216,7 @@ func ReaderContains(r io.Reader, subslice []byte) bool {
// GetTitleFunc returns a func that can be used to transform a string to
// title case.
//
// The supported styles are
// # The supported styles are
//
// - "Go" (strings.Title)
// - "AP" (see https://www.apstylebook.com/)
Expand Down Expand Up @@ -523,20 +520,3 @@ func PrintFs(fs afero.Fs, path string, w io.Writer) {
return nil
})
}

// HashString returns a hash from the given elements.
// It will panic if the hash cannot be calculated.
func HashString(elements ...any) string {
var o any
if len(elements) == 1 {
o = elements[0]
} else {
o = elements
}

hash, err := hashstructure.Hash(o, nil)
if err != nil {
panic(err)
}
return strconv.FormatUint(hash, 10)
}
7 changes: 0 additions & 7 deletions helpers/general_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -458,10 +458,3 @@ func BenchmarkUniqueStrings(b *testing.B) {
}
})
}

func TestHashString(t *testing.T) {
c := qt.New(t)

c.Assert(HashString("a", "b"), qt.Equals, "2712570657419664240")
c.Assert(HashString("ab"), qt.Equals, "590647783936702392")
}
24 changes: 24 additions & 0 deletions hugolib/page_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (

"github.com/bep/clock"
"github.com/gohugoio/hugo/htesting"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/markup/asciidocext"
"github.com/gohugoio/hugo/markup/rst"
"github.com/gohugoio/hugo/tpl"
Expand Down Expand Up @@ -2001,3 +2002,26 @@ Page1: {{ $p1.Path }}

b.AssertFileContent("public/index.html", "Lang: no", filepath.FromSlash("Page1: a/B/C/Page1.md"))
}

func TestPageHashString(t *testing.T) {
files := `
-- content/p1.md --
---
title: "p1"
---
-- content/p2.md --
---
title: "p2"
---
`

b := NewIntegrationTestBuilder(IntegrationTestConfig{
T: t,
TxtarString: files,
}).Build()

p1 := b.H.Sites[0].RegularPages()[0]
p2 := b.H.Sites[0].RegularPages()[1]

b.Assert(identity.HashString(p1), qt.Not(qt.Equals), identity.HashString(p2))
}
5 changes: 2 additions & 3 deletions hugolib/resource_chain_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,10 @@ import (
"testing"
"time"

"github.com/gohugoio/hugo/helpers"

qt "github.com/frankban/quicktest"

"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/resources/resource_transformers/tocss/scss"
)

Expand Down Expand Up @@ -122,7 +121,7 @@ FAILED REMOTE ERROR DETAILS CONTENT: |failed to fetch remote resource: Internal
|StatusCode: 500|ContentLength: 16|ContentType: text/plain; charset=utf-8|
`, helpers.HashString(ts.URL+"/sunset.jpg", map[string]any{})))
`, identity.HashString(ts.URL+"/sunset.jpg", map[string]any{})))

b.AssertFileContent("public/styles.min.a1df58687c3c9cc38bf26532f7b4b2f2c2b0315dcde212376959995c04f11fef.css", "body{background-color:#add8e6}")
b.AssertFileContent("public//styles2.min.1cfc52986836405d37f9998a63fd6dd8608e8c410e5e3db1daaa30f78bc273ba.css", "body{background-color:orange}")
Expand Down
13 changes: 13 additions & 0 deletions identity/identity.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,16 @@
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package identity

import (
Expand Down
63 changes: 63 additions & 0 deletions identity/identityhash.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package identity

import (
"strconv"

"github.com/mitchellh/hashstructure"
)

// HashString returns a hash from the given elements.
// It will panic if the hash cannot be calculated.
// Note that this hash should be used primarily for identity, not for change detection as
// it in the more complex values (e.g. Page) will not hash the full content.
func HashString(elements ...any) string {
hash := HashUint64(elements...)
return strconv.FormatUint(hash, 10)
}

// HashUint64 returns a hash from the given elements.
// It will panic if the hash cannot be calculated.
// Note that this hash should be used primarily for identity, not for change detection as
// it in the more complex values (e.g. Page) will not hash the full content.
func HashUint64(elements ...any) uint64 {
// For structs, hashstructure.Hash only works on the exported fields,
// so rewrite the input slice for known identity types.
for i, e := range elements {
switch t := e.(type) {
case Identity:
elements[i] = t.GetIdentity()
case keyer:
elements[i] = t.Key()
}
}

var o any
if len(elements) == 1 {
o = elements[0]
} else {
o = elements
}

hash, err := hashstructure.Hash(o, nil)
if err != nil {
panic(err)
}
return hash
}

type keyer interface {
Key() string
}
27 changes: 27 additions & 0 deletions identity/identityhash_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
// Copyright 2023 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package identity

import (
"testing"

qt "github.com/frankban/quicktest"
)

func TestHashString(t *testing.T) {
c := qt.New(t)

c.Assert(HashString("a", "b"), qt.Equals, "2712570657419664240")
c.Assert(HashString("ab"), qt.Equals, "590647783936702392")
}
4 changes: 2 additions & 2 deletions metrics/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import (

"github.com/gohugoio/hugo/common/types"
"github.com/gohugoio/hugo/compare"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/identity"
)

// The Provider interface defines an interface for measuring metrics.
Expand Down Expand Up @@ -242,7 +242,7 @@ func howSimilar(a, b any) int {
return 90
}

h1, h2 := helpers.HashString(a), helpers.HashString(b)
h1, h2 := identity.HashString(a), identity.HashString(b)
if h1 == h2 {
return 100
}
Expand Down
5 changes: 3 additions & 2 deletions resources/image.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (
color_extractor "github.com/marekm4/color-extractor"

"github.com/gohugoio/hugo/common/paths"
"github.com/gohugoio/hugo/identity"

"github.com/disintegration/gift"

Expand Down Expand Up @@ -278,7 +279,7 @@ func (i *imageResource) Filter(filters ...any) (images.ImageResource, error) {
gfilters = append(gfilters, images.ToFilters(f)...)
}

conf.Key = helpers.HashString(gfilters)
conf.Key = identity.HashString(gfilters)
conf.TargetFormat = i.Format

return i.doWithImageConfig(conf, func(src image.Image) (image.Image, error) {
Expand Down Expand Up @@ -430,7 +431,7 @@ func (i *imageResource) getImageMetaCacheTargetPath() string {
}
p1, _ := paths.FileAndExt(df.file)
h, _ := i.hash()
idStr := helpers.HashString(h, i.size(), imageMetaVersionNumber, cfgHash)
idStr := identity.HashString(h, i.size(), imageMetaVersionNumber, cfgHash)
p := path.Join(df.dir, fmt.Sprintf("%s_%s.json", p1, idStr))
return p
}
Expand Down
4 changes: 2 additions & 2 deletions resources/images/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import (
"strconv"
"strings"

"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/media"

"errors"
Expand Down Expand Up @@ -139,7 +139,7 @@ func DecodeConfig(m map[string]any) (ImagingConfig, error) {

i := ImagingConfig{
Cfg: defaultImaging,
CfgHash: helpers.HashString(m),
CfgHash: identity.HashString(m),
}

if err := mapstructure.WeakDecode(m, &i.Cfg); err != nil {
Expand Down
11 changes: 5 additions & 6 deletions resources/images/filters_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,18 +16,17 @@ package images
import (
"testing"

"github.com/gohugoio/hugo/helpers"

qt "github.com/frankban/quicktest"
"github.com/gohugoio/hugo/identity"
)

func TestFilterHash(t *testing.T) {
c := qt.New(t)

f := &Filters{}

c.Assert(helpers.HashString(f.Grayscale()), qt.Equals, helpers.HashString(f.Grayscale()))
c.Assert(helpers.HashString(f.Grayscale()), qt.Not(qt.Equals), helpers.HashString(f.Invert()))
c.Assert(helpers.HashString(f.Gamma(32)), qt.Not(qt.Equals), helpers.HashString(f.Gamma(33)))
c.Assert(helpers.HashString(f.Gamma(32)), qt.Equals, helpers.HashString(f.Gamma(32)))
c.Assert(identity.HashString(f.Grayscale()), qt.Equals, identity.HashString(f.Grayscale()))
c.Assert(identity.HashString(f.Grayscale()), qt.Not(qt.Equals), identity.HashString(f.Invert()))
c.Assert(identity.HashString(f.Gamma(32)), qt.Not(qt.Equals), identity.HashString(f.Gamma(33)))
c.Assert(identity.HashString(f.Gamma(32)), qt.Equals, identity.HashString(f.Gamma(32)))
}
4 changes: 2 additions & 2 deletions resources/internal/key.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

package internal

import "github.com/gohugoio/hugo/helpers"
import "github.com/gohugoio/hugo/identity"

// ResourceTransformationKey are provided by the different transformation implementations.
// It identifies the transformation (name) and its configuration (elements).
Expand All @@ -38,5 +38,5 @@ func (k ResourceTransformationKey) Value() string {
return k.Name
}

return k.Name + "_" + helpers.HashString(k.elements...)
return k.Name + "_" + identity.HashString(k.elements...)
}
6 changes: 3 additions & 3 deletions resources/resource_factories/create/remote.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import (
"github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/common/types"
"github.com/gohugoio/hugo/helpers"
"github.com/gohugoio/hugo/identity"
"github.com/gohugoio/hugo/media"
"github.com/gohugoio/hugo/resources"
"github.com/gohugoio/hugo/resources/resource"
Expand Down Expand Up @@ -235,9 +235,9 @@ func (c *Client) validateFromRemoteArgs(uri string, options fromRemoteOptions) e

func calculateResourceID(uri string, optionsm map[string]any) string {
if key, found := maps.LookupEqualFold(optionsm, "key"); found {
return helpers.HashString(key)
return identity.HashString(key)
}
return helpers.HashString(uri, optionsm)
return identity.HashString(uri, optionsm)
}

func addDefaultHeaders(req *http.Request) {
Expand Down
Loading

0 comments on commit 8219147

Please sign in to comment.