Consolidate most memory into a LRU cache

This commit also consolidates all (or the most important) memory caches in Hugo.

Fixes gohugoio#7425
Fixes gohugoio#7437
Fixes gohugoio#7436
Updates gohugoio#7544

cache/memcache/memcache_test.go

@@ -0,0 +1,205 @@
+// Copyright 2020 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package memcache
+
+import (
+	"fmt"
+	"path/filepath"
+	"sync"
+	"testing"
+	"time"
+
+	qt "github.com/frankban/quicktest"
+)
+
+func TestCache(t *testing.T) {
+	t.Parallel()
+	c := qt.New(t)
+
+	cache := New(Config{})
+
+	counter := 0
+	create := func() (interface{}, error) {
+		counter++
+		return counter, nil
+	}
+
+	for i := 0; i < 5; i++ {
+		v1, err := cache.getOrCreate("a", "a1", create)
+		c.Assert(err, qt.IsNil)
+		c.Assert(v1, qt.Equals, 1)
+		v2, err := cache.getOrCreate("a", "a2", create)
+		c.Assert(err, qt.IsNil)
+		c.Assert(v2, qt.Equals, 2)
+	}
+
+	cache.Clear()
+
+	v3, err := cache.getOrCreate("a", "a2", create)
+	c.Assert(err, qt.IsNil)
+	c.Assert(v3, qt.Equals, 3)
+}
+
+func TestCacheConcurrent(t *testing.T) {
+	t.Parallel()
+
+	c := qt.New(t)
+
+	var wg sync.WaitGroup
+
+	cache := New(Config{})
+
+	create := func(i int) func() (interface{}, error) {
+		return func() (interface{}, error) {
+			return i, nil
+		}
+	}
+
+	for i := 0; i < 10; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for j := 0; j < 100; j++ {
+				id := fmt.Sprintf("id%d", j)
+				v, err := cache.getOrCreate("a", id, create(j))
+				c.Assert(err, qt.IsNil)
+				c.Assert(v, qt.Equals, j)
+			}
+		}()
+	}
+	wg.Wait()
+}
+
+func TestCacheMemStats(t *testing.T) {
+	t.Parallel()
+	c := qt.New(t)
+
+	cache := New(Config{
+		ItemsToPrune:  10,
+		CheckInterval: 500 * time.Millisecond,
+	})
+
+	s := cache.stats
+
+	c.Assert(s.memstatsStart.Alloc > 0, qt.Equals, true)
+	c.Assert(s.memstatsCurrent.Alloc, qt.Equals, uint64(0))
+	c.Assert(s.availableMemory > 0, qt.Equals, true)
+	c.Assert(s.numItems, qt.Equals, uint64(0))
+
+	counter := 0
+	create := func() (interface{}, error) {
+		counter++
+		return counter, nil
+	}
+
+	for i := 1; i <= 20; i++ {
+		_, err := cache.getOrCreate("a", fmt.Sprintf("b%d", i), create)
+		c.Assert(err, qt.IsNil)
+	}
+
+	c.Assert(s.numItems, qt.Equals, uint64(20))
+	cache.cache.SetMaxSize(10)
+	time.Sleep(time.Millisecond * 600)
+	c.Assert(int(s.numItems), qt.Equals, 10)
+
+	c.Assert(s.memstatsCurrent.Alloc > 0, qt.Equals, true)
+}
+
+func TestDeleteRelated(t *testing.T) {
+	t.Parallel()
+	c := qt.New(t)
+
+	create := func() (interface{}, error) {
+		return "value", nil
+	}
+
+	mustGetOrCreate := func(g Getter, path string) {
+		_, err := g.GetOrCreate(path, create)
+		c.Assert(err, qt.IsNil)
+	}
+
+	createAndFillCache := func() *Cache {
+		cache := New(Config{
+			Running: true,
+		})
+
+		p1 := cache.GetOrCreatePartition("p1", ClearNever)
+		p2 := cache.GetOrCreatePartition("p2", ClearNever)
+
+		mustGetOrCreate(p1, "a/b/c.json")
+		mustGetOrCreate(p1, "a/b/d.js")
+		mustGetOrCreate(p1, "b/c.txt")
+		mustGetOrCreate(p1, "c/d.doc")
+		mustGetOrCreate(p1, "ROOT/d.js")
+		mustGetOrCreate(p1, "ROOT/d.json")
+		mustGetOrCreate(p1, "ROOT/e.txt")
+		mustGetOrCreate(p2, "f/b/e.js")
+
+		return cache
+
+	}
+
+	cache := createAndFillCache()
+	c.Assert(cache.DeleteRelatedTo("c/c.doc"), qt.Equals, 1)
+
+	cache = createAndFillCache()
+	c.Assert(cache.DeleteRelatedTo("a/b/d.js"), qt.Equals, 5)
+
+	cache = createAndFillCache()
+	c.Assert(cache.DeleteRelatedTo("ROOT/foo.bar"), qt.Equals, 6)
+
+}
+
+func TestSplitBasePathAndExt(t *testing.T) {
+	t.Parallel()
+	c := qt.New(t)
+
+	tests := []struct {
+		path string
+		a    string
+		b    string
+	}{
+		{"a/b.json", "a", "json"},
+		{"a/b/c/d.json", "a", "json"},
+	}
+	for i, this := range tests {
+		msg := qt.Commentf("test %d", i)
+		a, b := splitBasePathAndExt(this.path)
+
+		c.Assert(a, qt.Equals, this.a, msg)
+		c.Assert(b, qt.Equals, this.b, msg)
+	}
+
+}
+
+func TestCleanKey(t *testing.T) {
+	c := qt.New(t)
+
+	c.Assert(CleanKey(filepath.FromSlash("a/b/c.js")), qt.Equals, "a/b/c.js")
+	c.Assert(CleanKey("a//b////c.js"), qt.Equals, "a/b/c.js")
+	c.Assert(CleanKey("a.js"), qt.Equals, "CACHE_ROOT/a.js")
+	c.Assert(CleanKey("b/a"), qt.Equals, "b/a.UNKNOWN")
+
+}
+
+func TestKeyValid(t *testing.T) {
+	c := qt.New(t)
+
+	c.Assert(keyValid("a/b.j"), qt.Equals, true)
+	c.Assert(keyValid("a/b."), qt.Equals, false)
+	c.Assert(keyValid("a/b"), qt.Equals, false)
+	c.Assert(keyValid("/a/b.txt"), qt.Equals, false)
+	c.Assert(keyValid("a\\b.js"), qt.Equals, false)
+
+}