file_server: support precompressed files

modules/caddyhttp/encode/br/brotli_precompressed.go

@@ -0,0 +1,31 @@
+package caddybr
+
+import (
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp/encode"
+)
+
+func init() {
+	caddy.RegisterModule(BrotliPrecompressed{})
+}
+
+// BrotliPrecompressed provides the file extension for files precompressed with brotli encoding
+type BrotliPrecompressed struct{}
+
+// CaddyModule returns the Caddy module information.
+func (BrotliPrecompressed) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID:  "http.precompressed.br",
+		New: func() caddy.Module { return new(BrotliPrecompressed) },
+	}
+}
+
+// AcceptEncoding returns the name of the encoding as
+// used in the Accept-Encoding request headers.
+func (BrotliPrecompressed) AcceptEncoding() string { return "br" }
+
+// Suffix returns the filename suffix of precomressed files
+func (BrotliPrecompressed) Suffix() string { return "br" }
+
+// Interface guards
+var _ encode.Precompressed = (*BrotliPrecompressed)(nil)

modules/caddyhttp/encode/caddyfile.go

@@ -97,7 +97,7 @@ func (enc *Encode) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
 				}
 				enc.Types = types
 			default:
-				modID := "http.encoders." + name
+				modID := "http.precompressed." + name
 				unm, err := caddyfile.UnmarshalModule(d, modID)
 				if err != nil {
 					return err

modules/caddyhttp/encode/encode.go

@@ -127,7 +127,7 @@ func (enc *Encode) Validate() error {
 }
 
 func (enc *Encode) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
-	for _, encName := range acceptedEncodings(r, enc.Prefer) {
+	for _, encName := range AcceptedEncodings(r, enc.Prefer) {
 		if _, ok := enc.writerPools[encName]; !ok {
 			continue // encoding not offered
 		}
@@ -313,14 +313,14 @@ func (rw *responseWriter) init() {
 	rw.Header().Del("Accept-Ranges") // we don't know ranges for dynamically-encoded content
 }
 
-// acceptedEncodings returns the list of encodings that the
+// AcceptedEncodings returns the list of encodings that the
 // client supports, in descending order of preference.
 // The client preference via q-factor and the server
 // preference via Prefer setting are taken into account. If
 // the Sec-WebSocket-Key header is present then non-identity
 // encodings are not considered. See
 // http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html.
-func acceptedEncodings(r *http.Request, preferredOrder []string) []string {
+func AcceptedEncodings(r *http.Request, preferredOrder []string) []string {
 	acceptEncHeader := r.Header.Get("Accept-Encoding")
 	websocketKey := r.Header.Get("Sec-WebSocket-Key")
 	if acceptEncHeader == "" {
@@ -409,6 +409,13 @@ type Encoding interface {
 	NewEncoder() Encoder
 }
 
+// Precompressed is a type which returns filename suffix of precomressed
+// file and the name used in the Accept-Encoding header.
+type Precompressed interface {
+	AcceptEncoding() string
+	Suffix() string
+}
+
 var bufPool = sync.Pool{
 	New: func() interface{} {
 		return new(bytes.Buffer)

modules/caddyhttp/encode/encode_test.go

@@ -112,9 +112,9 @@ func TestPreferOrder(t *testing.T) {
 				r.Header.Set("Accept-Encoding", test.accept)
 			}
 			enc.Prefer = test.prefer
-			result := acceptedEncodings(r, enc.Prefer)
+			result := AcceptedEncodings(r, enc.Prefer)
 			if !sliceEqual(result, test.expected) {
-				t.Errorf("acceptedEncodings() actual: %s expected: %s",
+				t.Errorf("AcceptedEncodings() actual: %s expected: %s",
 					result,
 					test.expected)
 			}

modules/caddyhttp/encode/gzip/gzip_precompressed.go

@@ -0,0 +1,29 @@
+package caddygzip
+
+import (
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp/encode"
+)
+
+func init() {
+	caddy.RegisterModule(GzipPrecompressed{})
+}
+
+// GzipPrecompressed provides the file extension for files precompressed with gzip encoding
+type GzipPrecompressed struct {
+	Gzip
+}
+
+// CaddyModule returns the Caddy module information.
+func (GzipPrecompressed) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID:  "http.precompressed.gzip",
+		New: func() caddy.Module { return new(GzipPrecompressed) },
+	}
+}
+
+// Suffix returns the filename suffix of precomressed files
+func (GzipPrecompressed) Suffix() string { return "gz" }
+
+// Interface guards
+var _ encode.Precompressed = (*GzipPrecompressed)(nil)

modules/caddyhttp/encode/zstd/zstd_precompressed.go

@@ -0,0 +1,29 @@
+package caddyzstd
+
+import (
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp/encode"
+)
+
+func init() {
+	caddy.RegisterModule(ZstdPrecompressed{})
+}
+
+// ZstdPrecompressed provides the file extension for files precompressed with zstandard encoding
+type ZstdPrecompressed struct {
+	Zstd
+}
+
+// CaddyModule returns the Caddy module information.
+func (ZstdPrecompressed) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID:  "http.precompressed.zstd",
+		New: func() caddy.Module { return new(ZstdPrecompressed) },
+	}
+}
+
+// Suffix returns the filename suffix of precomressed files
+func (ZstdPrecompressed) Suffix() string { return "zst" }
+
+// Interface guards
+var _ encode.Precompressed = (*ZstdPrecompressed)(nil)

modules/caddyhttp/fileserver/caddyfile.go

@@ -15,12 +15,15 @@
 package fileserver
 
 import (
+	"fmt"
 	"path/filepath"
 	"strings"
 
 	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/caddyconfig"
 	"github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp/encode"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp/rewrite"
 )
 
@@ -33,10 +36,11 @@ func init() {
 // server and configures it with this syntax:
 //
 //    file_server [<matcher>] [browse] {
-//        root   <path>
-//	      hide   <files...>
-//	      index  <files...>
-//	      browse [<template_file>]
+//        root          <path>
+//        hide          <files...>
+//        index         <files...>
+//        browse        [<template_file>]
+//        precompressed <formats...>
 //    }
 //
 func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) {
@@ -77,6 +81,26 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
 				}
 				fsrv.Browse = new(Browse)
 				h.Args(&fsrv.Browse.TemplateFile)
+			case "precompressed":
+				var order []string
+				for h.NextArg() {
+					modID := "http.precompressed." + h.Val()
+					mod, err := caddy.GetModule(modID)
+					if err != nil {
+						return nil, h.Errf("getting module named '%s': %v", modID, err)
+					}
+					inst := mod.New()
+					precompress, ok := inst.(encode.Precompressed)
+					if !ok {
+						return nil, fmt.Errorf("module %s is not an HTTP encoding; is %T", modID, inst)
+					}
+					if fsrv.PrecompressRaw == nil {
+						fsrv.PrecompressRaw = make(caddy.ModuleMap)
+					}
+					fsrv.PrecompressRaw[h.Val()] = caddyconfig.JSON(precompress, nil)
+					order = append(order, h.Val())
+				}
+				fsrv.PrecompressOrder = order
 			default:
 				return nil, h.Errf("unknown subdirective '%s'", h.Val())
 			}

modules/caddyhttp/fileserver/staticfiles.go

@@ -31,6 +31,7 @@ import (
 
 	"github.com/caddyserver/caddy/v2"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp/encode"
 	"go.uber.org/zap"
 )
 
@@ -79,6 +80,14 @@ type FileServer struct {
 	// a 404 error. By default, this is false (disabled).
 	PassThru bool `json:"pass_thru,omitempty"`
 
+	// Selection of encoders to use to check for precompressed files.
+	PrecompressRaw caddy.ModuleMap `json:"encodings,omitempty" caddy:"namespace=http.precompressed"`
+
+	// If the client has no strong preference, choose these encodings in order.
+	PrecompressOrder []string
+
+	precompressors map[string]encode.Precompressed
+
 	logger *zap.Logger
 }
 
@@ -129,6 +138,32 @@ func (fsrv *FileServer) Provision(ctx caddy.Context) error {
 		}
 	}
 
+	mods, err := ctx.LoadModule(fsrv, "PrecompressRaw")
+	if err != nil {
+		return fmt.Errorf("loading encoder modules: %v", err)
+	}
+	for modName, modIface := range mods.(map[string]interface{}) {
+		p, ok := modIface.(encode.Precompressed)
+		if !ok {
+			return fmt.Errorf("module %s is not precompressor", modName)
+		}
+		ae := p.AcceptEncoding()
+		if ae == "" {
+			return fmt.Errorf("precompressor does not specify an Accept-Encoding value")
+		}
+		suffix := p.Suffix()
+		if suffix == "" {
+			return fmt.Errorf("precompressor does not specify a Suffix value")
+		}
+		if _, ok := fsrv.precompressors[ae]; ok {
+			return fmt.Errorf("precompressor already added: %s", ae)
+		}
+		if fsrv.precompressors == nil {
+			fsrv.precompressors = make(map[string]encode.Precompressed)
+		}
+		fsrv.precompressors[ae] = p
+	}
+
 	return nil
 }
 
@@ -206,8 +241,6 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 		return fsrv.notFound(w, r, next)
 	}
 
-	// TODO: content negotiation (brotli sidecar files, etc...)
-
 	// one last check to ensure the file isn't hidden (we might
 	// have changed the filename from when we last checked)
 	if fileHidden(filename, filesToHide) {
@@ -231,18 +264,49 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
 		}
 	}
 
-	fsrv.logger.Debug("opening file", zap.String("filename", filename))
+	var file *os.File
 
-	// open the file
-	file, err := fsrv.openFile(filename, w)
-	if err != nil {
-		if herr, ok := err.(caddyhttp.HandlerError); ok &&
-			herr.StatusCode == http.StatusNotFound {
-			return fsrv.notFound(w, r, next)
+	// check for precompressed files
+	for _, ae := range encode.AcceptedEncodings(r, fsrv.PrecompressOrder) {
+		precompress, ok := fsrv.precompressors[ae]
+		if !ok {
+			continue
+		}
+		compressedFilename := filename + "." + precompress.Suffix()
+		compressedInfo, err := os.Stat(compressedFilename)
+		if err != nil || compressedInfo.IsDir() {
+			fsrv.logger.Debug("precompressed file not accessable", zap.String("filename", compressedFilename))
+			continue
+		}
+		fsrv.logger.Debug("opening file", zap.String("filename", compressedFilename))
+		file, err = fsrv.openFile(compressedFilename, w)
+		if err != nil {
+			fsrv.logger.Warn("opening precompressed file failed", zap.String("filename", compressedFilename))
+			if caddyErr, ok := err.(caddyhttp.HandlerError); ok && caddyErr.StatusCode == http.StatusServiceUnavailable {
+				return err
+			}
+			continue
+		}
+		defer file.Close()
+		w.Header().Set("Content-Encoding", ae)
+		break
+	}
+
+	// no precompressed file found, use the actual file
+	if file == nil {
+		fsrv.logger.Debug("opening file", zap.String("filename", filename))
+
+		// open the file
+		file, err = fsrv.openFile(filename, w)
+		if err != nil {
+			if herr, ok := err.(caddyhttp.HandlerError); ok &&
+				herr.StatusCode == http.StatusNotFound {
+				return fsrv.notFound(w, r, next)
+			}
+			return err // error is already structured
 		}
-		return err // error is already structured
+		defer file.Close()
 	}
-	defer file.Close()
 
 	// set the ETag - note that a conditional If-None-Match request is handled
 	// by http.ServeContent below, which checks against this ETag value

modules/caddyhttp/standard/imports.go

@@ -5,6 +5,7 @@ import (
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyauth"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode"
+	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode/br"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode/gzip"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode/zstd"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/fileserver"