Share HTML template renderers and create a watcher framework (#20218)

The recovery, API, Web and package frameworks all create their own HTML
Renderers. This increases the memory requirements of Gitea
unnecessarily with duplicate templates being kept in memory.

Further the reloading framework in dev mode for these involves locking
and recompiling all of the templates on each load. This will potentially
hide concurrency issues and it is inefficient.

This PR stores the templates renderer in the context and stores this
context in the NormalRoutes, it then creates a fsnotify.Watcher
framework to watch files.

The watching framework is then extended to the mailer templates which
were previously not being reloaded in dev.

Then the locales are simplified to a similar structure.

Fix #20210 
Fix #20211
Fix #20217

Signed-off-by: Andrew Thornton <art27@cantab.net>

cmd/embedded.go

@@ -123,7 +123,7 @@ func initEmbeddedExtractor(c *cli.Context) error {
 
 	sections["public"] = &section{Path: "public", Names: public.AssetNames, IsDir: public.AssetIsDir, Asset: public.Asset}
 	sections["options"] = &section{Path: "options", Names: options.AssetNames, IsDir: options.AssetIsDir, Asset: options.Asset}
-	sections["templates"] = &section{Path: "templates", Names: templates.AssetNames, IsDir: templates.AssetIsDir, Asset: templates.Asset}
+	sections["templates"] = &section{Path: "templates", Names: templates.BuiltinAssetNames, IsDir: templates.BuiltinAssetIsDir, Asset: templates.BuiltinAsset}
 
 	for _, sec := range sections {
 		assets = append(assets, buildAssetList(sec, pats, c)...)

cmd/web.go

@@ -126,8 +126,10 @@ func runWeb(ctx *cli.Context) error {
 				return err
 			}
 		}
-		c := install.Routes()
+		installCtx, cancel := context.WithCancel(graceful.GetManager().HammerContext())
+		c := install.Routes(installCtx)
 		err := listen(c, false)
+		cancel()
 		if err != nil {
 			log.Critical("Unable to open listener for installer. Is Gitea already running?")
 			graceful.GetManager().DoGracefulShutdown()
@@ -175,7 +177,7 @@ func runWeb(ctx *cli.Context) error {
 	}
 
 	// Set up Chi routes
-	c := routers.NormalRoutes()
+	c := routers.NormalRoutes(graceful.GetManager().HammerContext())
 	err := listen(c, true)
 	<-graceful.GetManager().Done()
 	log.Info("PID: %d Gitea Web Finished", os.Getpid())

contrib/pr/checkout.go

@@ -27,6 +27,7 @@ import (
 	"code.gitea.io/gitea/models/db"
 	"code.gitea.io/gitea/models/unittest"
 	gitea_git "code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/graceful"
 	"code.gitea.io/gitea/modules/markup"
 	"code.gitea.io/gitea/modules/markup/external"
 	repo_module "code.gitea.io/gitea/modules/repository"
@@ -117,7 +118,7 @@ func runPR() {
 	// routers.GlobalInit()
 	external.RegisterRenderers()
 	markup.Init()
-	c := routers.NormalRoutes()
+	c := routers.NormalRoutes(graceful.GetManager().HammerContext())
 
 	log.Printf("[PR] Ready for testing !\n")
 	log.Printf("[PR] Login with user1, user2, user3, ... with pass: password\n")

go.mod

@@ -28,6 +28,7 @@ require (
 	github.com/emirpasic/gods v1.18.1
 	github.com/ethantkoenig/rupture v1.0.1
 	github.com/felixge/fgprof v0.9.2
+	github.com/fsnotify/fsnotify v1.5.4
 	github.com/gliderlabs/ssh v0.3.4
 	github.com/go-ap/activitypub v0.0.0-20220615144428-48208c70483b
 	github.com/go-ap/jsonld v0.0.0-20220615144122-1d862b15410d
@@ -161,7 +162,6 @@ require (
 	github.com/envoyproxy/protoc-gen-validate v0.6.2 // indirect
 	github.com/felixge/httpsnoop v1.0.2 // indirect
 	github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect
-	github.com/fsnotify/fsnotify v1.5.4 // indirect
 	github.com/fullstorydev/grpcurl v1.8.1 // indirect
 	github.com/fxamacker/cbor/v2 v2.4.0 // indirect
 	github.com/go-ap/errors v0.0.0-20220615144307-e8bc4a40ae9f // indirect

integrations/api_activitypub_person_test.go

@@ -23,10 +23,10 @@ import (
 
 func TestActivityPubPerson(t *testing.T) {
 	setting.Federation.Enabled = true
-	c = routers.NormalRoutes()
+	c = routers.NormalRoutes(context.TODO())
 	defer func() {
 		setting.Federation.Enabled = false
-		c = routers.NormalRoutes()
+		c = routers.NormalRoutes(context.TODO())
 	}()
 
 	onGiteaRun(t, func(*testing.T, *url.URL) {
@@ -60,10 +60,10 @@ func TestActivityPubPerson(t *testing.T) {
 
 func TestActivityPubMissingPerson(t *testing.T) {
 	setting.Federation.Enabled = true
-	c = routers.NormalRoutes()
+	c = routers.NormalRoutes(context.TODO())
 	defer func() {
 		setting.Federation.Enabled = false
-		c = routers.NormalRoutes()
+		c = routers.NormalRoutes(context.TODO())
 	}()
 
 	onGiteaRun(t, func(*testing.T, *url.URL) {
@@ -75,10 +75,10 @@ func TestActivityPubMissingPerson(t *testing.T) {
 
 func TestActivityPubPersonInbox(t *testing.T) {
 	setting.Federation.Enabled = true
-	c = routers.NormalRoutes()
+	c = routers.NormalRoutes(context.TODO())
 	defer func() {
 		setting.Federation.Enabled = false
-		c = routers.NormalRoutes()
+		c = routers.NormalRoutes(context.TODO())
 	}()
 
 	srv := httptest.NewServer(c)

integrations/api_nodeinfo_test.go

@@ -5,6 +5,7 @@
 package integrations
 
 import (
+	"context"
 	"net/http"
 	"net/url"
 	"testing"
@@ -18,10 +19,10 @@ import (
 
 func TestNodeinfo(t *testing.T) {
 	setting.Federation.Enabled = true
-	c = routers.NormalRoutes()
+	c = routers.NormalRoutes(context.TODO())
 	defer func() {
 		setting.Federation.Enabled = false
-		c = routers.NormalRoutes()
+		c = routers.NormalRoutes(context.TODO())
 	}()
 
 	onGiteaRun(t, func(*testing.T, *url.URL) {

integrations/create_no_session_test.go

@@ -5,6 +5,7 @@
 package integrations
 
 import (
+	"context"
 	"net/http"
 	"net/http/httptest"
 	"os"
@@ -57,7 +58,7 @@ func TestSessionFileCreation(t *testing.T) {
 	oldSessionConfig := setting.SessionConfig.ProviderConfig
 	defer func() {
 		setting.SessionConfig.ProviderConfig = oldSessionConfig
-		c = routers.NormalRoutes()
+		c = routers.NormalRoutes(context.TODO())
 	}()
 
 	var config session.Options
@@ -82,7 +83,7 @@ func TestSessionFileCreation(t *testing.T) {
 
 	setting.SessionConfig.ProviderConfig = string(newConfigBytes)
 
-	c = routers.NormalRoutes()
+	c = routers.NormalRoutes(context.TODO())
 
 	t.Run("NoSessionOnViewIssue", func(t *testing.T) {
 		defer PrintCurrentTest(t)()

integrations/integration_test.go

@@ -89,7 +89,7 @@ func TestMain(m *testing.M) {
 	defer cancel()
 
 	initIntegrationTest()
-	c = routers.NormalRoutes()
+	c = routers.NormalRoutes(context.TODO())
 
 	// integration test settings...
 	if setting.Cfg != nil {

modules/charset/escape_test.go

@@ -133,11 +133,18 @@ then resh (ר), and finally heh (ה) (which should appear leftmost).`,
 	},
 }
 
+type nullLocale struct{}
+
+func (nullLocale) Language() string                                                   { return "" }
+func (nullLocale) Tr(key string, _ ...interface{}) string                             { return key }
+func (nullLocale) TrN(cnt interface{}, key1, keyN string, args ...interface{}) string { return "" }
+
+var _ (translation.Locale) = nullLocale{}
+
 func TestEscapeControlString(t *testing.T) {
 	for _, tt := range escapeControlTests {
 		t.Run(tt.name, func(t *testing.T) {
-			locale := translation.NewLocale("en_US")
-			status, result := EscapeControlString(tt.text, locale)
+			status, result := EscapeControlString(tt.text, nullLocale{})
 			if !reflect.DeepEqual(*status, tt.status) {
 				t.Errorf("EscapeControlString() status = %v, wanted= %v", status, tt.status)
 			}
@@ -173,7 +180,7 @@ func TestEscapeControlReader(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			input := strings.NewReader(tt.text)
 			output := &strings.Builder{}
-			status, err := EscapeControlReader(input, output, translation.NewLocale("en_US"))
+			status, err := EscapeControlReader(input, output, nullLocale{})
 			result := output.String()
 			if err != nil {
 				t.Errorf("EscapeControlReader(): err = %v", err)
@@ -195,5 +202,5 @@ func TestEscapeControlReader_panic(t *testing.T) {
 	for i := 0; i < 6826; i++ {
 		bs = append(bs, []byte("—")...)
 	}
-	_, _ = EscapeControlString(string(bs), translation.NewLocale("en_US"))
+	_, _ = EscapeControlString(string(bs), nullLocale{})
 }

modules/context/context.go

@@ -658,8 +658,8 @@ func Auth(authMethod auth.Method) func(*Context) {
 }
 
 // Contexter initializes a classic context for a request.
-func Contexter() func(next http.Handler) http.Handler {
-	rnd := templates.HTMLRenderer()
+func Contexter(ctx context.Context) func(next http.Handler) http.Handler {
+	_, rnd := templates.HTMLRenderer(ctx)
 	csrfOpts := getCsrfOpts()
 	if !setting.IsProd {
 		CsrfTokenRegenerationInterval = 5 * time.Second // in dev, re-generate the tokens more aggressively for debug purpose

modules/context/package.go

@@ -5,6 +5,7 @@
 package context
 
 import (
+	gocontext "context"
 	"fmt"
 	"net/http"
 
@@ -14,6 +15,7 @@ import (
 	"code.gitea.io/gitea/models/unit"
 	user_model "code.gitea.io/gitea/models/user"
 	"code.gitea.io/gitea/modules/structs"
+	"code.gitea.io/gitea/modules/templates"
 )
 
 // Package contains owner, access mode and optional the package descriptor
@@ -118,12 +120,14 @@ func packageAssignment(ctx *Context, errCb func(int, string, interface{})) {
 }
 
 // PackageContexter initializes a package context for a request.
-func PackageContexter() func(next http.Handler) http.Handler {
+func PackageContexter(ctx gocontext.Context) func(next http.Handler) http.Handler {
+	_, rnd := templates.HTMLRenderer(ctx)
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
 			ctx := Context{
-				Resp: NewResponse(resp),
-				Data: map[string]interface{}{},
+				Resp:   NewResponse(resp),
+				Data:   map[string]interface{}{},
+				Render: rnd,
 			}
 			defer ctx.Close()
 

modules/options/base.go

@@ -0,0 +1,40 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package options
+
+import (
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+
+	"code.gitea.io/gitea/modules/util"
+)
+
+func walkAssetDir(root string, callback func(path, name string, d fs.DirEntry, err error) error) error {
+	if err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+		// name is the path relative to the root
+		name := path[len(root):]
+		if len(name) > 0 && name[0] == '/' {
+			name = name[1:]
+		}
+		if err != nil {
+			if os.IsNotExist(err) {
+				return callback(path, name, d, err)
+			}
+			return err
+		}
+		if util.CommonSkip(d.Name()) {
+			if d.IsDir() {
+				return fs.SkipDir
+			}
+			return nil
+		}
+		return callback(path, name, d, err)
+	}); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("unable to get files for assets in %s: %w", root, err)
+	}
+	return nil
+}

modules/options/dynamic.go

@@ -8,8 +8,10 @@ package options
 
 import (
 	"fmt"
+	"io/fs"
 	"os"
 	"path"
+	"path/filepath"
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
@@ -45,7 +47,7 @@ func Dir(name string) ([]string, error) {
 
 	isDir, err = util.IsDir(staticDir)
 	if err != nil {
-		return []string{}, fmt.Errorf("Unabe to check if static directory %s is a directory. %v", staticDir, err)
+		return []string{}, fmt.Errorf("unable to check if static directory %s is a directory. %v", staticDir, err)
 	}
 	if isDir {
 		files, err := util.StatDir(staticDir, true)
@@ -64,6 +66,18 @@ func Locale(name string) ([]byte, error) {
 	return fileFromDir(path.Join("locale", name))
 }
 
+// WalkLocales reads the content of a specific locale from static or custom path.
+func WalkLocales(callback func(path, name string, d fs.DirEntry, err error) error) error {
+	if err := walkAssetDir(filepath.Join(setting.StaticRootPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("failed to walk locales. Error: %w", err)
+	}
+
+	if err := walkAssetDir(filepath.Join(setting.CustomPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("failed to walk locales. Error: %w", err)
+	}
+	return nil
+}
+
 // Readme reads the content of a specific readme from static or custom path.
 func Readme(name string) ([]byte, error) {
 	return fileFromDir(path.Join("readme", name))

modules/options/static.go

@@ -9,8 +9,10 @@ package options
 import (
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path"
+	"path/filepath"
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
@@ -74,6 +76,14 @@ func Locale(name string) ([]byte, error) {
 	return fileFromDir(path.Join("locale", name))
 }
 
+// WalkLocales reads the content of a specific locale from static or custom path.
+func WalkLocales(callback func(path, name string, d fs.DirEntry, err error) error) error {
+	if err := walkAssetDir(filepath.Join(setting.CustomPath, "options", "locale"), callback); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("failed to walk locales. Error: %w", err)
+	}
+	return nil
+}
+
 // Readme reads the content of a specific readme from bindata or custom path.
 func Readme(name string) ([]byte, error) {
 	return fileFromDir(path.Join("readme", name))