Add ranged requests suport to transports

DarthSim · DarthSim · commit 58fd025f8966 · 2022-09-07T18:03:12.000+06:00
diff --git a/httprange/httprange.go b/httprange/httprange.go
@@ -0,0 +1,73 @@
+package httprange
+
+import (
+	"errors"
+	"net/http"
+	"net/textproto"
+	"strconv"
+	"strings"
+)
+
+func Parse(s string) (int64, int64, error) {
+	if s == "" {
+		return 0, 0, nil // header not present
+	}
+
+	const b = "bytes="
+	if !strings.HasPrefix(s, b) {
+		return 0, 0, errors.New("invalid range")
+	}
+
+	for _, ra := range strings.Split(s[len(b):], ",") {
+		ra = textproto.TrimString(ra)
+		if ra == "" {
+			continue
+		}
+
+		i := strings.Index(ra, "-")
+		if i < 0 {
+			return 0, 0, errors.New("invalid range")
+		}
+
+		start, end := textproto.TrimString(ra[:i]), textproto.TrimString(ra[i+1:])
+
+		if start == "" {
+			// Don't support ranges without start since it looks like FFmpeg doen't use ones
+			return 0, 0, errors.New("invalid range")
+		}
+
+		istart, err := strconv.ParseInt(start, 10, 64)
+		if err != nil || i < 0 {
+			return 0, 0, errors.New("invalid range")
+		}
+
+		var iend int64
+
+		if end == "" {
+			iend = -1
+		} else {
+			iend, err = strconv.ParseInt(end, 10, 64)
+			if err != nil || istart > iend {
+				return 0, 0, errors.New("invalid range")
+			}
+		}
+
+		return istart, iend, nil
+	}
+
+	return 0, 0, errors.New("invalid range")
+}
+
+func InvalidHTTPRangeResponse(req *http.Request) *http.Response {
+	return &http.Response{
+		StatusCode:    http.StatusRequestedRangeNotSatisfiable,
+		Proto:         "HTTP/1.0",
+		ProtoMajor:    1,
+		ProtoMinor:    0,
+		Header:        make(http.Header),
+		ContentLength: 0,
+		Body:          nil,
+		Close:         false,
+		Request:       req,
+	}
+}
diff --git a/transport/azure/azure.go b/transport/azure/azure.go
@@ -8,6 +8,7 @@ import (
 
 	"github.com/Azure/azure-storage-blob-go/azblob"
 	"github.com/imgproxy/imgproxy/v3/config"
+	"github.com/imgproxy/imgproxy/v3/httprange"
 )
 
 type transport struct {
@@ -40,12 +41,22 @@ func (t transport) RoundTrip(req *http.Request) (resp *http.Response, err error)
 	containerURL := t.serviceURL.NewContainerURL(strings.ToLower(req.URL.Host))
 	blobURL := containerURL.NewBlockBlobURL(strings.TrimPrefix(req.URL.Path, "/"))
 
-	get, err := blobURL.Download(req.Context(), 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
+	start, end, err := httprange.Parse(req.Header.Get("Range"))
+	if err != nil {
+		return httprange.InvalidHTTPRangeResponse(req), nil
+	}
+
+	length := end - start + 1
+	if end <= 0 {
+		length = azblob.CountToEnd
+	}
+
+	get, err := blobURL.Download(req.Context(), start, length, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{})
 	if err != nil {
 		return nil, err
 	}
 
-	if config.ETagEnabled {
+	if config.ETagEnabled && start == 0 && end == azblob.CountToEnd {
 		etag := string(get.ETag())
 
 		if etag == req.Header.Get("If-None-Match") {
diff --git a/transport/fs/file_limiter.go b/transport/fs/file_limiter.go
@@ -0,0 +1,27 @@
+package fs
+
+import (
+	"io"
+	"net/http"
+)
+
+type fileLimiter struct {
+	f    http.File
+	left int
+}
+
+func (lr *fileLimiter) Read(p []byte) (n int, err error) {
+	if lr.left <= 0 {
+		return 0, io.EOF
+	}
+	if len(p) > lr.left {
+		p = p[0:lr.left]
+	}
+	n, err = lr.f.Read(p)
+	lr.left -= n
+	return
+}
+
+func (lr *fileLimiter) Close() error {
+	return lr.f.Close()
+}
diff --git a/transport/fs/fs.go b/transport/fs/fs.go
@@ -6,11 +6,15 @@ import (
 	"fmt"
 	"io"
 	"io/fs"
+	"mime"
 	"net/http"
 	"os"
+	"path/filepath"
+	"strconv"
 	"strings"
 
 	"github.com/imgproxy/imgproxy/v3/config"
+	"github.com/imgproxy/imgproxy/v3/httprange"
 )
 
 type transport struct {
@@ -41,7 +45,32 @@ func (t transport) RoundTrip(req *http.Request) (resp *http.Response, err error)
 		return respNotFound(req, fmt.Sprintf("%s is directory", req.URL.Path)), nil
 	}
 
-	if config.ETagEnabled {
+	statusCode := 200
+	size := fi.Size()
+	body := io.ReadCloser(f)
+
+	mime := mime.TypeByExtension(filepath.Ext(fi.Name()))
+	header.Set("Content-Type", mime)
+
+	start, end, err := httprange.Parse(req.Header.Get("Range"))
+	switch {
+	case err != nil:
+		f.Close()
+		return httprange.InvalidHTTPRangeResponse(req), nil
+
+	case end != 0:
+		if end < 0 {
+			end = size - 1
+		}
+
+		f.Seek(start, io.SeekStart)
+
+		statusCode = http.StatusPartialContent
+		size = end - start + 1
+		body = &fileLimiter{f: f, left: int(size)}
+		header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, fi.Size()))
+
+	case config.ETagEnabled:
 		etag := BuildEtag(req.URL.Path, fi)
 		header.Set("ETag", etag)
 
@@ -62,15 +91,16 @@ func (t transport) RoundTrip(req *http.Request) (resp *http.Response, err error)
 		}
 	}
 
+	header.Set("Content-Length", strconv.Itoa(int(size)))
+
 	return &http.Response{
-		Status:        "200 OK",
-		StatusCode:    200,
+		StatusCode:    statusCode,
 		Proto:         "HTTP/1.0",
 		ProtoMajor:    1,
 		ProtoMinor:    0,
 		Header:        header,
-		ContentLength: fi.Size(),
-		Body:          f,
+		ContentLength: size,
+		Body:          body,
 		Close:         true,
 		Request:       req,
 	}, nil
diff --git a/transport/gcs/gcs.go b/transport/gcs/gcs.go
@@ -12,6 +12,7 @@ import (
 	"google.golang.org/api/option"
 
 	"github.com/imgproxy/imgproxy/v3/config"
+	"github.com/imgproxy/imgproxy/v3/httprange"
 )
 
 // For tests
@@ -58,40 +59,82 @@ func (t transport) RoundTrip(req *http.Request) (*http.Response, error) {
 		obj = obj.Generation(g)
 	}
 
+	var (
+		reader     *storage.Reader
+		statusCode int
+		size       int64
+	)
+
 	header := make(http.Header)
 
-	if config.ETagEnabled {
-		attrs, err := obj.Attrs(req.Context())
+	if r := req.Header.Get("Range"); len(r) != 0 {
+		start, end, err := httprange.Parse(r)
 		if err != nil {
-			return handleError(req, err)
+			return httprange.InvalidHTTPRangeResponse(req), nil
 		}
-		header.Set("ETag", attrs.Etag)
-
-		if etag := req.Header.Get("If-None-Match"); len(etag) > 0 && attrs.Etag == etag {
-			return &http.Response{
-				StatusCode:    http.StatusNotModified,
-				Proto:         "HTTP/1.0",
-				ProtoMajor:    1,
-				ProtoMinor:    0,
-				Header:        header,
-				ContentLength: 0,
-				Body:          nil,
-				Close:         false,
-				Request:       req,
-			}, nil
+
+		if end != 0 {
+			length := end - start + 1
+			if end < 0 {
+				length = -1
+			}
+
+			reader, err = obj.NewRangeReader(req.Context(), start, length)
+			if err != nil {
+				return nil, err
+			}
+
+			if end < 0 || end >= reader.Attrs.Size {
+				end = reader.Attrs.Size - 1
+			}
+
+			size = end - reader.Attrs.StartOffset + 1
+
+			statusCode = http.StatusPartialContent
+			header.Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", reader.Attrs.StartOffset, end, reader.Attrs.Size))
 		}
 	}
 
-	reader, err := obj.NewReader(req.Context())
-	if err != nil {
-		return handleError(req, err)
+	// We haven't initialize reader yet, this means that we need non-ranged reader
+	if reader == nil {
+		if config.ETagEnabled {
+			attrs, err := obj.Attrs(req.Context())
+			if err != nil {
+				return handleError(req, err)
+			}
+			header.Set("ETag", attrs.Etag)
+
+			if etag := req.Header.Get("If-None-Match"); len(etag) > 0 && attrs.Etag == etag {
+				return &http.Response{
+					StatusCode:    http.StatusNotModified,
+					Proto:         "HTTP/1.0",
+					ProtoMajor:    1,
+					ProtoMinor:    0,
+					Header:        header,
+					ContentLength: 0,
+					Body:          nil,
+					Close:         false,
+					Request:       req,
+				}, nil
+			}
+		}
+
+		var err error
+		reader, err = obj.NewReader(req.Context())
+		if err != nil {
+			return handleError(req, err)
+		}
+
+		statusCode = 200
+		size = reader.Attrs.Size
 	}
 
+	header.Set("Content-Length", strconv.Itoa(int(size)))
+	header.Set("Content-Type", reader.Attrs.ContentType)
 	header.Set("Cache-Control", reader.Attrs.CacheControl)
 
 	return &http.Response{
-		Status:        "200 OK",
-		StatusCode:    200,
+		StatusCode:    statusCode,
 		Proto:         "HTTP/1.0",
 		ProtoMajor:    1,
 		ProtoMinor:    0,
diff --git a/transport/s3/s3.go b/transport/s3/s3.go
@@ -53,7 +53,9 @@ func (t transport) RoundTrip(req *http.Request) (resp *http.Response, err error)
 		input.VersionId = aws.String(req.URL.RawQuery)
 	}
 
-	if config.ETagEnabled {
+	if r := req.Header.Get("Range"); len(r) != 0 {
+		input.Range = aws.String(r)
+	} else if config.ETagEnabled {
 		if ifNoneMatch := req.Header.Get("If-None-Match"); len(ifNoneMatch) > 0 {
 			input.IfNoneMatch = aws.String(ifNoneMatch)
 		}
diff --git a/transport/swift/swift.go b/transport/swift/swift.go
@@ -46,7 +46,12 @@ func (t transport) RoundTrip(req *http.Request) (resp *http.Response, err error)
 	container := req.URL.Host
 	objectName := strings.TrimPrefix(req.URL.Path, "/")
 
-	object, objectHeaders, err := t.con.ObjectOpen(req.Context(), container, objectName, false, make(swift.Headers))
+	reqHeaders := make(swift.Headers)
+	if r := req.Header.Get("Range"); len(r) > 0 {
+		reqHeaders["Range"] = r
+	}
+
+	object, objectHeaders, err := t.con.ObjectOpen(req.Context(), container, objectName, false, reqHeaders)
 
 	header := make(http.Header)
 

Original file line number	Diff line number	Diff line change
`@@ -53,7 +53,9 @@ func (t transport) RoundTrip(req http.Request) (resp http.Response, err error)`
`53`	`53`	`input.VersionId = aws.String(req.URL.RawQuery)`
`54`	`54`	`}`
`55`	`55`
`56`		`- if config.ETagEnabled {`
	`56`	`+ if r := req.Header.Get("Range"); len(r) != 0 {`
	`57`	`+ input.Range = aws.String(r)`
	`58`	`+ } else if config.ETagEnabled {`
`57`	`59`	`if ifNoneMatch := req.Header.Get("If-None-Match"); len(ifNoneMatch) > 0 {`
`58`	`60`	`input.IfNoneMatch = aws.String(ifNoneMatch)`
`59`	`61`	`}`