Handle more transforms in query type

Added more tests
Fixed s3 metadata setting and parsing

README.md

@@ -124,7 +124,7 @@ buckets: # list of available buckets
             secretAccessKey: "random"
         transform: # config for transforms
             path: "\\/(?P<presetName>[a-z0-9_]+)\\/(?P<parent>[a-z0-9-\\.]+)" # regexp for transform path 
-            kind: "presets" #  type of transform for now only "presets" is available 
+            kind: "presets" #  type of transform or "query"
             presets: # list of presets
                 small:
                     quality: 75

config/types.go

@@ -61,6 +61,7 @@ type Storage struct {
 	Region          string            `yaml:"region,omitempty"`          // region for s3 storage
 	Endpoint        string            `yaml:"endpoint,omitempty"`        // endpoint for s3 storage
 	PathPrefix      string            `yaml:"pathPrefix,omitempty"`      // prefix in path for all storage
+	Bucket          string            `yaml:"bucket"`
 	Hash            string            // unique hash for given storage
 }
 

configuration/config.yml

@@ -57,3 +57,22 @@ buckets:
                 kind: "local-meta"
                 rootPath: "/Users/aldor/workspace/mkaciubacom/web"
                 pathPrefix: "transforms"
+
+    query:
+        keys:
+          - accessKey: "acc"
+            secretAccessKey: "sec"
+        transform:
+            kind: "query"
+            resultKey: "hash"
+            presets:
+        storages:
+            basic:
+                kind: "http"
+                url: "https://i.imgur.com/<item>"
+                headers:
+                  "x--key": "sec"
+            transform:
+                kind: "local-meta"
+                rootPath: "/Users/aldor/workspace/mkaciubacom/web"
+                pathPrefix: "transforms"

engine/image_engine_test.go

@@ -0,0 +1,51 @@
+package engine
+
+import (
+	"github.com/aldor007/mort/config"
+	"github.com/aldor007/mort/object"
+	"github.com/aldor007/mort/response"
+	"github.com/aldor007/mort/transforms"
+	"github.com/stretchr/testify/assert"
+	"os"
+	"testing"
+)
+
+func TestImageEngine_Process_Error(t *testing.T) {
+	image := response.NewNoContent(500)
+	mortConfig := config.Config{}
+	mortConfig.Load("testdata/config.yml")
+	obj, err := object.NewFileObjectFromPath("/local/small/parent.jpg", &mortConfig)
+
+	assert.Nil(t, err)
+	assert.NotNil(t, obj)
+
+	e := NewImageEngine(image)
+	res, err := e.Process(obj, []transforms.Transforms{obj.Transforms})
+
+	assert.NotNil(t, err)
+	assert.Equal(t, res.StatusCode, 500)
+}
+
+func TestImageEngine_Process(t *testing.T) {
+	f, err := os.Open("testdata/small.jpg")
+	if err != nil {
+		panic(err)
+	}
+
+	image := response.New(200, f)
+	mortConfig := config.Config{}
+	mortConfig.Load("testdata/config.yml")
+	obj, err := object.NewFileObjectFromPath("/local/small/parent.jpg", &mortConfig)
+
+	assert.Nil(t, err)
+	assert.NotNil(t, obj)
+
+	e := NewImageEngine(image)
+	res, err := e.Process(obj, []transforms.Transforms{obj.Transforms})
+
+	assert.Nil(t, err)
+	assert.Equal(t, res.StatusCode, 200)
+	assert.Equal(t, res.Headers.Get("content-type"), "image/jpeg")
+	assert.Equal(t, res.Headers.Get("etag"), "4a4e9789cc1e902c")
+	assert.Equal(t, res.Headers.Get("x-amz-meta-public-width"), "300")
+}

engine/testdata/config.yml

@@ -0,0 +1,21 @@
+buckets:
+    local:
+        transform:
+            path: "\\/(?P<parent>[a-zA-Z0-9\\.\\/]+)\\-(?P<presetName>[a-z]+)"
+            kind: "presets"
+            parentBucket: "local"
+            presets:
+                small:
+                    quality: 75
+                    filters:
+                        thumbnail:
+                            width: 100
+                            height: 70
+                            mode: outbound
+                        interlace: true
+        storages:
+            basic:
+                kind: "local-meta"
+                rootPath: "../tests/benchmark"
+            transform:
+                kind: "noop"

log/logger_test.go

@@ -0,0 +1,17 @@
+package log
+
+import (
+	"go.uber.org/zap"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestLog(t *testing.T) {
+	l := zap.NewNop()
+	RegisterLogger(l)
+	assert.Equal(t, l, Log())
+
+	s := l.Sugar()
+	assert.Equal(t, s, Logs())
+}

middleware/s3.go

@@ -13,7 +13,7 @@ import (
 	"github.com/aldor007/go-aws-auth"
 )
 
-// authHeaderRegexpv4 reugular expression for AWS Auth v4 header mode
+// authHeaderRegexpv4 regular expression for AWS Auth v4 header mode
 var autHeaderRegexpv4 = regexp.MustCompile("^(:?[A-Za-z0-9-]+) Credential=(:?.+),\\s*SignedHeaders=(:?[a-zA-Z0-9;-]+),\\s*Signature=(:?[a-zA-Z0-9]+)$")
 
 // authHeaderRegexpv2 regular expression for Aws Auth v2 header mode

object/file_object.go

@@ -10,9 +10,10 @@ import (
 
 // FileObject is representing parsed request for image or file
 type FileObject struct {
-	Uri         *url.URL              `json:"uri"`        // original request path
-	Bucket      string                `json:"bucket"`     // request matched bucket
-	Key         string                `json:"key"`        // storage path for file
+	Uri         *url.URL `json:"uri"`    // original request path
+	Bucket      string   `json:"bucket"` // request matched bucket
+	Key         string   `json:"key"`    // storage path for file with leading slash
+	key         string
 	Transforms  transforms.Transforms `json:"transforms"` // list of transform that should be performed
 	Storage     config.Storage        `json:"storage"`    // selected storage that should be used
 	Parent      *FileObject           // original image for transformed image
@@ -33,7 +34,7 @@ func NewFileObjectFromPath(path string, mortConfig *config.Config) (*FileObject,
 	err := Parse(obj.Uri, mortConfig, &obj)
 
 	log.Log().Info("FileObject", zap.String("path", path), zap.String("key", obj.Key), zap.String("bucket", obj.Bucket), zap.String("storage", obj.Storage.Kind),
-		zap.Bool("hasTransforms", !obj.Transforms.NotEmpty), zap.Bool("hasParent", obj.HasParent()))
+		zap.Bool("hasTransforms", obj.HasTransform()), zap.Bool("hasParent", obj.HasParent()))
 	return &obj, err
 }
 
@@ -49,7 +50,7 @@ func NewFileObject(uri *url.URL, mortConfig *config.Config) (*FileObject, error)
 	err := Parse(uri, mortConfig, &obj)
 
 	log.Log().Info("FileObject", zap.String("path", uri.Path), zap.String("key", obj.Key), zap.String("bucket", obj.Bucket), zap.String("storage", obj.Storage.Kind),
-		zap.Bool("hasTransforms", !obj.Transforms.NotEmpty), zap.Bool("hasParent", obj.HasParent()))
+		zap.Bool("hasTransforms", obj.HasTransform()), zap.Bool("hasParent", obj.HasParent()))
 	return &obj, err
 }
 

object/preset.go

@@ -8,11 +8,9 @@ import (
 	"go.uber.org/zap"
 	"net/url"
 	"path"
-	"strconv"
-	"strings"
 )
 
-// presetCache cache use presets because we don't need create it always new for each request
+// presetCache cache used presets because we don't need create it always new for each request
 var presetCache = make(map[string]transforms.Transforms)
 
 // decodePreset parse given url by matching user defined regexp with request path
@@ -30,6 +28,7 @@ func decodePreset(url *url.URL, mortConfig *config.Config, bucketConfig config.B
 			subMatchMap[name] = matches[i]
 		}
 	}
+
 	presetName := subMatchMap["presetName"]
 	parent := subMatchMap["parent"]
 
@@ -53,8 +52,9 @@ func decodePreset(url *url.URL, mortConfig *config.Config, bucketConfig config.B
 	parentObj.Storage = bucketConfig.Storages.Get(trans.ParentStorage)
 
 	if parentObj != nil && bucketConfig.Transform.ResultKey == "hash" {
-		obj.Key = "/" + strings.Join([]string{strconv.FormatUint(uint64(obj.Transforms.Hash().Sum64()), 16), subMatchMap["parent"]}, "-")
+		obj.Key = hashKey(obj.Transforms.Hash(), subMatchMap["parent"])
 	}
+
 	obj.Parent = parentObj
 	obj.CheckParent = trans.CheckParent
 	return err

object/query.go

@@ -6,20 +6,22 @@ import (
 	"net/url"
 	"path"
 	"strconv"
-	"strings"
 )
 
 func decodeQuery(url *url.URL, mortConfig *config.Config, bucketConfig config.Bucket, obj *FileObject) error {
 	trans := bucketConfig.Transform
-	parent := obj.Key
 
 	var err error
 	obj.Transforms, err = queryToTransform(url.Query())
 
-	parent = "/" + path.Join(trans.ParentBucket, parent)
-
 	if obj.HasTransform() {
-		obj.Key = "/" + strings.Join([]string{strconv.FormatUint(uint64(obj.Transforms.Hash().Sum64()), 16), parent}, "-")
+		parentBucket := obj.Bucket
+		if trans.ParentBucket != "" {
+			parentBucket = trans.ParentBucket
+		}
+
+		parent := "/" + path.Join(parentBucket, obj.Key)
+		obj.Key = hashKey(obj.Transforms.Hash(), obj.key)
 		parentObj, err := NewFileObjectFromPath(parent, mortConfig)
 		parentObj.Storage = bucketConfig.Storages.Get(trans.ParentStorage)
 		obj.Parent = parentObj
@@ -36,22 +38,31 @@ func queryToTransform(query url.Values) (transforms.Transforms, error) {
 		return trans, nil
 	}
 
+	var err error
 	opt := query.Get("operation")
 	if opt == "" {
-		opt = "resize"
-	}
-
-	var err error
-	switch opt {
-	case "resize":
 		err = trans.Resize(queryToInt(query, "width"), queryToInt(query, "height"), false)
-	case "crop":
-		err = trans.Crop(queryToInt(query, "width"), queryToInt(query, "height"), false)
+	} else {
+		for qsKey, values := range query {
+			if qsKey == "operation" {
+				for _, o := range values {
+					switch o {
+					case "resize":
+						err = trans.Resize(queryToInt(query, "width"), queryToInt(query, "height"), false)
+					case "crop":
+						err = trans.Crop(queryToInt(query, "width"), queryToInt(query, "height"), false)
+					case "watermark":
+						var sigma float64
+						sigma, err = strconv.ParseFloat(query.Get("sigma"), 32)
+						err = trans.Watermark(query.Get("image"), query.Get("position"), float32(sigma))
+					}
+				}
+			}
+		}
 	}
-	//case "watermark":
-	//opacity, err := strnconv. query.Get("opacity")
-	//err = trans.Watermark(query.Get("image"), query.Get("position"),
+
 	err = trans.Quality(queryToInt(query, "quality"))
+	err = trans.Format(query.Get("format"))
 
 	return trans, err
 }

object/uri.go

@@ -1,12 +1,24 @@
 package object
 
 import (
+	"bytes"
 	"errors"
 	"github.com/aldor007/mort/config"
+	"hash"
 	"net/url"
+	"strconv"
 	"strings"
+	"sync"
 )
 
+var bufPool = sync.Pool{
+	New: func() interface{} {
+		return &bytes.Buffer{}
+	},
+}
+
+var bytePrefix = []byte("prefix")
+
 // Parse pare given url using appropriate parser
 // it set object Key, Bucket, Parent and transforms
 func Parse(url *url.URL, mortConfig *config.Config, obj *FileObject) error {
@@ -15,6 +27,7 @@ func Parse(url *url.URL, mortConfig *config.Config, obj *FileObject) error {
 	obj.Bucket = elements[1]
 	if len(elements) > 2 {
 		obj.Key = "/" + elements[2]
+		obj.key = elements[2]
 	}
 
 	if bucketConfig, ok := mortConfig.Buckets[obj.Bucket]; ok {
@@ -41,3 +54,17 @@ func Parse(url *url.URL, mortConfig *config.Config, obj *FileObject) error {
 
 	return errors.New("unknown bucket")
 }
+
+func hashKey(h hash.Hash64, suffix string) string {
+	hashB := []byte(strconv.FormatUint(uint64(h.Sum64()), 16))
+	buf := bufPool.Get().(*bytes.Buffer)
+	buf.Reset()
+	buf.WriteByte('/')
+	buf.Write(hashB[0:3])
+	buf.WriteByte('/')
+	buf.Write(hashB)
+	buf.WriteByte('-')
+	buf.WriteString(suffix)
+	defer bufPool.Put(buf)
+	return buf.String()
+}

package.json

@@ -1,10 +1,11 @@
 {
-  "name": "github.com/aldor007/mort-integrations-tests",
+  "name": "mort-integrations-tests",
   "version": "1.0.0",
   "dependencies": {
     "aws-sdk": "^2.131.0",
     "chai": "^4.1.2",
-    "supertest": "^3.0.0"
+    "supertest": "^3.0.0",
+    "mocha": "^4.0.1"
   },
   "devDependencies": {
     "mocha": "^4.0.1"

processor/processor_test.go

@@ -6,11 +6,30 @@ import (
 	"github.com/aldor007/mort/lock"
 	"github.com/aldor007/mort/object"
 	"github.com/aldor007/mort/throttler"
+	"github.com/stretchr/testify/assert"
 	"io/ioutil"
 	"net/http"
 	"testing"
 )
 
+func TestNewRequestProcessor(t *testing.T) {
+	req, _ := http.NewRequest("GET", "http://mort/local/small.jpg-small", nil)
+
+	config := config.Config{}
+	err := config.Load("../tests/benchmark/small.yml")
+	assert.Nil(t, err)
+
+	obj, err := object.NewFileObject(req.URL, &config)
+	assert.Nil(t, err)
+
+	rp := NewRequestProcessor(3, lock.NewMemoryLock(), throttler.NewBucketThrottler(10))
+	res := rp.Process(req, obj)
+
+	assert.Equal(t, res.StatusCode, 200)
+	assert.Equal(t, res.Headers.Get("x-amz-meta-public-width"), "105")
+	assert.Equal(t, res.Headers.Get("ETag"), "5b95df244105a698")
+}
+
 func BenchmarkNewRequestProcessorMemoryLock(b *testing.B) {
 	benchmarks := []struct {
 		name       string

processor/testdata/small.yml

@@ -0,0 +1,21 @@
+buckets:
+    local:
+        transform:
+            path: "\\/(?P<parent>[a-zA-Z0-9\\.\\/]+)\\-(?P<presetName>[a-z]+)"
+            kind: "presets"
+            parentBucket: "local"
+            presets:
+                small:
+                    quality: 75
+                    filters:
+                        thumbnail:
+                            width: 100
+                            height: 70
+                            mode: outbound
+                        interlace: true
+        storages:
+            basic:
+                kind: "local-meta"
+                rootPath: "./tests/benchmark"
+            transform:
+                kind: "noop"