Merge pull request #3 from klauspost/switch-gzip-pkg

Switch compression package (3x faster)

go.mod

@@ -4,6 +4,7 @@ go 1.13
 
 require (
 	github.com/gin-gonic/gin v1.5.0
+	github.com/klauspost/compress v1.11.3
 	github.com/signalsciences/ac v1.1.0
 	github.com/stretchr/testify v1.4.0
 )

go.sum

@@ -14,6 +14,8 @@ github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/klauspost/compress v1.11.3 h1:dB4Bn0tN3wdCzQxnS8r06kV74qN/TAfaIS0bVE8h3jc=
+github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/leodido/go-urn v1.1.0 h1:Sm1gr51B1kKyfD2BlRcLSiEkffoG96g6TPv6eRoEiB8=
 github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw=
 github.com/mattn/go-isatty v0.0.9 h1:d5US/mDsogSGW37IV293h//ZFaeajb69h+EHFsv2xGg=

handler.go

@@ -2,14 +2,14 @@ package gzip
 
 import (
 	"bufio"
-	"compress/gzip"
 	"fmt"
 	"io/ioutil"
 	"net"
 	"net/http"
 	"sync"
 
 	"github.com/gin-gonic/gin"
+	"github.com/klauspost/compress/gzip"
 )
 
 // These constants are copied from the gzip package
@@ -19,12 +19,18 @@ const (
 	BestCompression    = gzip.BestCompression
 	DefaultCompression = gzip.DefaultCompression
 	HuffmanOnly        = gzip.HuffmanOnly
+	// Stateless will do compression but without maintaining any state
+	// between Write calls, so long running responses will not take memory.
+	// There will be no memory kept between Write calls,
+	// but compression and speed will be suboptimal.
+	// Because of this, the size of actual Write calls will affect output size.
+	Stateless = gzip.StatelessCompression
 )
 
 // Config is used in Handler initialization
 type Config struct {
 	// gzip compression level to use,
-	// valid value: -2 ~ 9.
+	// valid value: -3 => 9.
 	//
 	// see https://golang.org/pkg/compress/gzip/#NewWriterLevel
 	CompressionLevel int
@@ -58,7 +64,7 @@ type Handler struct {
 //
 // config must not be modified after calling on NewHandler()
 func NewHandler(config Config) *Handler {
-	if config.CompressionLevel < HuffmanOnly || config.CompressionLevel > BestCompression {
+	if config.CompressionLevel < Stateless || config.CompressionLevel > BestCompression {
 		panic(fmt.Sprintf("gzip: invalid CompressionLevel: %d", config.CompressionLevel))
 	}
 	if config.MinContentLength <= 0 {

handler_test.go

@@ -113,7 +113,7 @@ func TestNewHandler_Checks(t *testing.T) {
 
 	assert.Panics(t, func() {
 		NewHandler(Config{
-			CompressionLevel: -3,
+			CompressionLevel: -4,
 			MinContentLength: 100,
 		})
 	})
@@ -150,7 +150,12 @@ func BenchmarkSoleGin_SmallPayload(b *testing.B) {
 	r.Header.Set("Accept-Encoding", "gzip")
 
 	b.ResetTimer()
+	h := map[string][]string(w.header)
 	for i := 0; i < b.N; i++ {
+		// Delete header between calls.
+		for k := range h {
+			delete(h, k)
+		}
 		g.ServeHTTP(w, r)
 	}
 
@@ -170,7 +175,12 @@ func BenchmarkGinWithDefaultHandler_SmallPayload(b *testing.B) {
 	r.Header.Set("Accept-Encoding", "gzip")
 
 	b.ResetTimer()
+	h := map[string][]string(w.header)
 	for i := 0; i < b.N; i++ {
+		// Delete header between calls.
+		for k := range h {
+			delete(h, k)
+		}
 		g.ServeHTTP(w, r)
 	}
 
@@ -190,7 +200,12 @@ func BenchmarkSoleGin_BigPayload(b *testing.B) {
 	r.Header.Set("Accept-Encoding", "gzip")
 
 	b.ResetTimer()
+	h := map[string][]string(w.header)
 	for i := 0; i < b.N; i++ {
+		// Delete header between calls.
+		for k := range h {
+			delete(h, k)
+		}
 		g.ServeHTTP(w, r)
 	}
 
@@ -210,7 +225,12 @@ func BenchmarkGinWithDefaultHandler_BigPayload(b *testing.B) {
 	r.Header.Set("Accept-Encoding", "gzip")
 
 	b.ResetTimer()
+	h := map[string][]string(w.header)
 	for i := 0; i < b.N; i++ {
+		// Delete header between calls.
+		for k := range h {
+			delete(h, k)
+		}
 		g.ServeHTTP(w, r)
 	}
 
@@ -265,6 +285,39 @@ func TestGinWithDefaultHandler(t *testing.T) {
 	}
 }
 
+func TestGinWithLevelsHandler(t *testing.T) {
+	for i := Stateless; i < 10; i++ {
+		var seq = "level_" + strconv.Itoa(i)
+		i := i
+		t.Run(seq, func(t *testing.T) {
+			g := newEchoGinInstance(bigPayload, NewHandler(Config{
+				CompressionLevel: i,
+				MinContentLength: 1,
+			}).Gin)
+
+			var (
+				w = httptest.NewRecorder()
+				r = httptest.NewRequest(http.MethodPost, "/", strings.NewReader(seq))
+			)
+
+			r.Header.Set("Accept-Encoding", "gzip")
+			g.ServeHTTP(w, r)
+
+			result := w.Result()
+			require.EqualValues(t, http.StatusOK, result.StatusCode)
+			require.Equal(t, "gzip", result.Header.Get("Content-Encoding"))
+			comp, err := ioutil.ReadAll(result.Body)
+			require.NoError(t, err)
+			reader, err := gzip.NewReader(bytes.NewReader(comp))
+			require.NoError(t, err)
+			body, err := ioutil.ReadAll(reader)
+			require.NoError(t, err)
+			require.True(t, bytes.HasPrefix(body, []byte(seq)))
+			t.Logf("%s: compressed %d => %d", seq, len(body), len(comp))
+		})
+	}
+}
+
 func TestGinWithDefaultHandler_404(t *testing.T) {
 	var (
 		g = newGinInstance(bigPayload, DefaultHandler().Gin)

writerwrapper.go

@@ -1,11 +1,12 @@
 package gzip
 
 import (
-	"compress/gzip"
 	"fmt"
 	"net/http"
 	"strconv"
 	"strings"
+
+	"github.com/klauspost/compress/gzip"
 )
 
 // writerWrapper wraps the originalHandler
@@ -154,10 +155,12 @@ func (w *writerWrapper) Write(data []byte) (int, error) {
 
 		w.WriteHeaderNow()
 		w.initGzipWriter()
-		written, err := w.gzipWriter.Write(w.bodyBuffer)
-		if err != nil {
-			err = fmt.Errorf("w.gzipWriter.Write: %w", err)
-			return written, err
+		if len(w.bodyBuffer) > 0 {
+			written, err := w.gzipWriter.Write(w.bodyBuffer)
+			if err != nil {
+				err = fmt.Errorf("w.gzipWriter.Write: %w", err)
+				return written, err
+			}
 		}
 		return w.gzipWriter.Write(data)
 	}

writerwrapper_test.go

@@ -1,14 +1,14 @@
 package gzip
 
 import (
-	"compress/gzip"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"strconv"
 	"sync"
 	"testing"
 
+	"github.com/klauspost/compress/gzip"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )