From 8669ca9a122c48b57d672f399a9270d3b08579c9 Mon Sep 17 00:00:00 2001 From: Marcin Kaciuba Date: Sun, 21 Apr 2019 14:23:08 +0200 Subject: [PATCH] feat: unit tests form exract --- Dockerfile | 10 +- cmd/mort/mort.go | 7 ++ doc/Configuration.md | 8 +- doc/Image-Operations.md | 64 ++++++++++- go.mod | 40 ++++--- go.sum | 144 +++++++++++++++++++++++++ pkg/cache/cache.go | 4 + pkg/cache/memory.go | 5 + pkg/cache/memory_test.go | 37 +++++++ pkg/cache/redis.go | 5 + pkg/config/config.go | 10 +- pkg/object/file_object.go | 9 ++ pkg/object/file_object_test.go | 1 + pkg/object/query.go | 9 ++ pkg/storage/storage.go | 10 ++ tests-int/Images.Spec.js | 191 +++++++++------------------------ tests-int/config.yml | 6 ++ 17 files changed, 385 insertions(+), 175 deletions(-) create mode 100644 go.sum create mode 100644 pkg/cache/memory_test.go diff --git a/Dockerfile b/Dockerfile index 808c9b2..b4aefdf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,8 +2,8 @@ FROM ubuntu:18.04 as builder # ENV LIBVIPS_VERSION 8.7.3 ENV LIBVIPS_VERSION 8.6.2 -ENV DEP_VERSION v0.5.0 -ENV GOLANG_VERSION 1.11 +ENV DEP_VERSION v0.5.1 +ENV GOLANG_VERSION 1.12.1 # Installs libvips + required libraries RUN \ @@ -39,16 +39,16 @@ RUN curl -fsSL --insecure "$GOLANG_DOWNLOAD_URL" -o golang.tar.gz \ && tar -C /usr/local -xzf golang.tar.gz \ && rm golang.tar.gz -ENV GOPATH /go +# ENV GOPATH /go ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH # ENV GOROOT /go:$GOROOT RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" WORKDIR $GOPATH -RUN curl -fsSL -o /usr/local/bin/dep https://github.com/golang/dep/releases/download/$DEP_VERSION/dep-linux-amd64 && chmod +x /usr/local/bin/dep +# RUN curl -fsSL -o /usr/local/bin/dep https://github.com/golang/dep/releases/download/$DEP_VERSION/dep-linux-amd64 && chmod +x /usr/local/bin/dep ADD . /go/src/github.com/aldor007/mort -RUN cd /go/src/github.com/aldor007/mort && dep ensure -vendor-only +# RUN cd /go/src/github.com/aldor007/mort && dep ensure -vendor-only RUN cd /go/src/github.com/aldor007/mort; go build -o /go/mort cmd/mort/mort.go; FROM ubuntu:18.04 diff --git a/cmd/mort/mort.go b/cmd/mort/mort.go index 43c2305..9b64acb 100644 --- a/cmd/mort/mort.go +++ b/cmd/mort/mort.go @@ -136,6 +136,13 @@ func configureMonitoring(mortConfig *config.Config) { []string{"method", "storage"}, )) + p.RegisterCounterVec("storage_request", prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "mort_storage_request", + Help: "mort requests storage", + }, + []string{"method", "bucket", "storage", "object_type"}, + )) + p.RegisterCounter("collapsed_count", prometheus.NewCounter(prometheus.CounterOpts{ Name: "mort_request_collapsed_count", Help: "mort count of collapsed requests", diff --git a/doc/Configuration.md b/doc/Configuration.md index 391c1df..269cf03 100644 --- a/doc/Configuration.md +++ b/doc/Configuration.md @@ -60,7 +60,13 @@ Server section describe configuration for HTTP server and some runtime variables server: listen: "0.0.0.0:8080" # default traffic listener monitoring: "" # default no monitoring ( or prometheus) - cacheSize: 10 # default size of cache used for new images + cache: + type: "memory" # default or redis + maxCacheItemSizeMB: 50 # max item size to cache default 5 MB + # config for redis + address: + - "localhost:6379" + clientConfig: # change redis instance config requestTimeout: 70 # default request timeout in seconds internalListen: "0.0.0.0:8081" # default listener for debug /debug and metrics /metrics plugins: # list of additional plugins diff --git a/doc/Image-Operations.md b/doc/Image-Operations.md index 354636f..fa9b6d0 100644 --- a/doc/Image-Operations.md +++ b/doc/Image-Operations.md @@ -19,12 +19,18 @@ Table of contents * [Crop](#crop) + [Preset](#preset-4) + [Query string](#query-string-4) - * [Watermark](#watermark) + * [Extract](#extract) + [Preset](#preset-5) + [Query string](#query-string-5) - * [Image format](#image-format) + * [ResizeCropAuto](#resizeCropAuto) + [Preset](#preset-6) + [Query string](#query-string-6) + * [Watermark](#watermark) + + [Preset](#preset-7) + + [Query string](#query-string-7) + * [Image format](#image-format) + + [Preset](#preset-8) + + [Query string](#query-string-8) ## Originals @@ -176,13 +182,63 @@ Position can be one of:
- + +## Extract + +Extract - crop given area from the image. +Parameters: +* top - point on x axis +* left - point on y axis +* width - width of the cropped area. +* height - height of the cropped area. + +### Preset + + +
+ +

crop image with width 500
+
+ + +### Query string + + +
+ +

crop image with width 200 and height 200
+
+ + +
+ +## ResizeCropAuto + +resize and crop given area from the image. Do it using proportion (center of image is used) +Parameters: +* width - width of the cropped area. +* height - height of the cropped area. + +### Preset + + +
+ +

crop image with width 555
+
+ + +### Query string + +
- +

crop image with width 200 and height 200
+
+ ## Watermark Add watermark to image diff --git a/go.mod b/go.mod index 0cf7b71..4e53a64 100644 --- a/go.mod +++ b/go.mod @@ -3,42 +3,40 @@ module github.com/aldor007/mort require ( github.com/aldor007/go-aws-auth v0.0.0-20180623204207-00898dfb9272 github.com/aldor007/stow v0.0.0-20190109171535-d59ae5fea5cf - github.com/aws/aws-sdk-go v1.16.14 - github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 - github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 - github.com/davecgh/go-spew v1.1.1 + github.com/aws/aws-sdk-go v1.16.14 // indirect + github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 // indirect github.com/djherbis/stream v1.2.0 github.com/go-chi/chi v3.3.3+incompatible github.com/go-redis/cache v6.3.5+incompatible github.com/go-redis/redis v6.15.2+incompatible - github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b - github.com/golang/protobuf v1.2.0 + github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect github.com/google/brotli v1.0.7 - github.com/google/readahead v0.0.0-20161222183148-eaceba169032 - github.com/graymeta/stow v0.0.0-20181228161447-b469cfb112f8 - github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af + github.com/google/readahead v0.0.0-20161222183148-eaceba169032 // indirect github.com/karlseguin/ccache v2.0.2+incompatible - github.com/kurin/blazer v0.5.1 - github.com/matttproud/golang_protobuf_extensions v1.0.1 + github.com/karlseguin/expect v1.0.1 // indirect + github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 // indirect + github.com/onsi/ginkgo v1.8.0 // indirect + github.com/onsi/gomega v1.5.0 // indirect github.com/pkg/errors v0.8.1 - github.com/pmezard/go-difflib v1.0.0 github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 - github.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7 + github.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7 // indirect github.com/prometheus/client_golang v0.9.2 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 - github.com/prometheus/common v0.0.0-20190107103113-2998b132700a - github.com/prometheus/procfs v0.0.0-20190104112138-b1a0a9a36d74 + github.com/prometheus/common v0.0.0-20190107103113-2998b132700a // indirect + github.com/prometheus/procfs v0.0.0-20190104112138-b1a0a9a36d74 // indirect + github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a // indirect github.com/spaolacci/murmur3 v0.0.0-20170819071325-9f5d223c6079 github.com/stretchr/testify v1.3.0 github.com/vmihailenco/msgpack v4.0.1+incompatible - go.uber.org/atomic v1.2.0 - go.uber.org/multierr v1.1.0 + github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 // indirect + go.uber.org/atomic v1.2.0 // indirect + go.uber.org/multierr v1.1.0 // indirect go.uber.org/zap v1.9.1 - golang.org/x/net v0.0.0-20190107155100-1a61f4433d85 - google.golang.org/appengine v1.4.0 + golang.org/x/net v0.0.0-20190311183353-d8887717615a + google.golang.org/appengine v1.4.0 // indirect gopkg.in/h2non/bimg.v1 v1.0.19 gopkg.in/h2non/gock.v1 v1.0.12 - gopkg.in/kothar/brotli-go.v0 v0.0.0-20170728081549-771231d473d6 - gopkg.in/kothar/go-backblaze.v0 v0.0.0-20180916190456-9ac0cf0dab1a + gopkg.in/karlseguin/expect.v1 v1.0.1 // indirect + gopkg.in/kothar/go-backblaze.v0 v0.0.0-20180916190456-9ac0cf0dab1a // indirect gopkg.in/yaml.v2 v2.2.2 ) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..a01a35a --- /dev/null +++ b/go.sum @@ -0,0 +1,144 @@ +github.com/aldor007/go-aws-auth v0.0.0-20180623204207-00898dfb9272 h1:TuxcUGZxsHKCW50twhDbGdRtuMXmMDCmOssevV0ZoOA= +github.com/aldor007/go-aws-auth v0.0.0-20180623204207-00898dfb9272/go.mod h1:gm7JFOl7Sp8r+lJqHXOkL0HyUFR05QXueqxVufqfkTg= +github.com/aldor007/stow v0.0.0-20190109171535-d59ae5fea5cf h1:wBAGFp3s42TlQe1mZIRlbIQaUz5y4TqpX/Stb0NoRLI= +github.com/aldor007/stow v0.0.0-20190109171535-d59ae5fea5cf/go.mod h1:TUBrYvdpmfMQKeKUIZeveJE/Y5VufnvvtDE/kxJ9X3U= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/aws/aws-sdk-go v1.16.14 h1:v/Jc1ryxAa6F9uPwXl2RSuFsGC4Iai8PEiSDP/cprHc= +github.com/aws/aws-sdk-go v1.16.14/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= +github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/djherbis/stream v1.2.0 h1:6KwMyps3yTU58GFfrHJwhuVfn+GU2MyajuLcncJcYCo= +github.com/djherbis/stream v1.2.0/go.mod h1:ZNVKPVRCmrwhCwQHZUpVHHrq2rtGLrG1t3T/TThYLP8= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-chi/chi v3.3.3+incompatible h1:KHkmBEMNkwKuK4FdQL7N2wOeB9jnIx7jR5wsuSBEFI8= +github.com/go-chi/chi v3.3.3+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-redis/cache v6.3.5+incompatible h1:4OUyoXXYRRQ6tKA4ue3TlPUkBzk3occzjtXBZBxCzgs= +github.com/go-redis/cache v6.3.5+incompatible/go.mod h1:XNnMdvlNjcZvHjsscEozHAeOeSE5riG9Fj54meG4WT4= +github.com/go-redis/redis v6.15.2+incompatible h1:9SpNVG76gr6InJGxoZ6IuuxaCOQwDAhzyXg+Bs+0Sb4= +github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/brotli v1.0.7 h1:fxwwohNEPaVS6qvtnjwgzRR62Upa70pkw0f9qarjrQs= +github.com/google/brotli v1.0.7/go.mod h1:XpGqLY1HgMKTQI5TU8iAKE/okaKqS9h1e6KRlRztlOU= +github.com/google/readahead v0.0.0-20161222183148-eaceba169032 h1:6Be3nkuJFyRfCgr6qTIzmRp8y9QwDIbqy/nYr9WDPos= +github.com/google/readahead v0.0.0-20161222183148-eaceba169032/go.mod h1:qYysrqQXuV4tzsizt4oOQ6mrBZQ0xnQXP3ylXX8Jk5Y= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/karlseguin/ccache v2.0.2+incompatible h1:MpSlLlHgG3vPWTAIJsSYlyAQsHwfQ2HzgUlbJFh9Ufk= +github.com/karlseguin/ccache v2.0.2+incompatible/go.mod h1:CM9tNPzT6EdRh14+jiW8mEF9mkNZuuE51qmgGYUB93w= +github.com/karlseguin/expect v1.0.1 h1:z4wy4npwwHSWKjGWH85WNJO42VQhovxTCZDSzhjo8hY= +github.com/karlseguin/expect v1.0.1/go.mod h1:zNBxMY8P21owkeogJELCLeHIt+voOSduHYTFUbwRAV8= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= +github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 h1:J9b7z+QKAmPf4YLrFg6oQUotqHQeUNWwkvo7jZp1GLU= +github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7 h1:gGBSHPOU7g8YjTbhwn+lvFm2VDEhhA+PwDIlstkgSxE= +github.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= +github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20190107103113-2998b132700a h1:bLKgQQEViHvsdgCwCGyyga8npETKygQ8b7c/28mJ8tw= +github.com/prometheus/common v0.0.0-20190107103113-2998b132700a/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190104112138-b1a0a9a36d74 h1:d1Xoc24yp/pXmWl2leBiBA+Tptce6cQsA+MMx/nOOcY= +github.com/prometheus/procfs v0.0.0-20190104112138-b1a0a9a36d74/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/spaolacci/murmur3 v0.0.0-20170819071325-9f5d223c6079 h1:lDiM+yMjW7Ork8mhl0YN0qO1Z02qGoe1vwzGc1gP/8U= +github.com/spaolacci/murmur3 v0.0.0-20170819071325-9f5d223c6079/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/vmihailenco/msgpack v4.0.1+incompatible h1:RMF1enSPeKTlXrXdOcqjFUElywVZjjC6pqse21bKbEU= +github.com/vmihailenco/msgpack v4.0.1+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= +github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 h1:3UeQBvD0TFrlVjOeLOBz+CPAI8dnbqNSVwUwRrkp7vQ= +github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= +go.uber.org/atomic v1.2.0 h1:yVVGhClJ8Xi1y4TxhJZE6QFPrz76BrzhWA01n47mSFk= +go.uber.org/atomic v1.2.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/h2non/bimg.v1 v1.0.19 h1:Li7mgBrIvCHvShB4nyCcGJ2Z2rWR/95kgI/U2+U2eYw= +gopkg.in/h2non/bimg.v1 v1.0.19/go.mod h1:PgsZL7dLwUbsGm1NYps320GxGgvQNTnecMCZqxV11So= +gopkg.in/h2non/gock.v1 v1.0.12 h1:o3JJqe+h7R9Ay6LtMeFrKz1WnokrJDrNpDQs9KGqVn8= +gopkg.in/h2non/gock.v1 v1.0.12/go.mod h1:KHI4Z1sxDW6P4N3DfTWSEza07YpkQP7KJBfglRMEjKY= +gopkg.in/karlseguin/expect.v1 v1.0.1 h1:9u0iUltnhFbJTHaSIH0EP+cuTU5rafIgmcsEsg2JQFw= +gopkg.in/karlseguin/expect.v1 v1.0.1/go.mod h1:uB7QIJBcclvYbwlUDkSCsGjAOMis3fP280LyhuDEf2I= +gopkg.in/kothar/go-backblaze.v0 v0.0.0-20180916190456-9ac0cf0dab1a h1:0sNUXDtpOFFPgaRPk5LOl9iDGsaVQOlLl2p8XFJrCNY= +gopkg.in/kothar/go-backblaze.v0 v0.0.0-20180916190456-9ac0cf0dab1a/go.mod h1:zJ2QpyDCYo1KvLXlmdnFlQAyF/Qfth0fB8239Qg7BIE= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 63f3963..c5d8ca0 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -6,12 +6,16 @@ import ( "github.com/aldor007/mort/pkg/response" ) + +// ResponseCache is interface for caching of mort responses type ResponseCache interface { Set(obj *object.FileObject, res *response.Response) error Get(obj *object.FileObject) (*response.Response, error) Delete(obj *object.FileObject) error } + +// Create returns instance of Response cache func Create(cacheCfg config.CacheCfg) ResponseCache { switch cacheCfg.Type { case "redis": diff --git a/pkg/cache/memory.go b/pkg/cache/memory.go index 7b45ada..8440ad9 100644 --- a/pkg/cache/memory.go +++ b/pkg/cache/memory.go @@ -10,14 +10,17 @@ import ( "time" ) +// MemoryCache uses memory for cache purpose type MemoryCache struct { cache *ccache.Cache // cache for created image transformations } +// NewMemoryCache returns instance of memory cache func NewMemoryCache(maxSize int64) *MemoryCache { return &MemoryCache{ccache.New(ccache.Configure().MaxSize(maxSize))} } +// Set put response to cache func (c *MemoryCache) Set(obj *object.FileObject, res *response.Response) error { monitoring.Report().Inc("cache_ratio;status:set") c.cache.Set(obj.GetResponseCacheKey(), res, time.Second*time.Duration(res.GetTTL())) @@ -25,6 +28,7 @@ func (c *MemoryCache) Set(obj *object.FileObject, res *response.Response) error return nil } +// Get returns instance from cache or error (if not found in cache) func (c *MemoryCache) Get(obj *object.FileObject) (*response.Response, error) { cacheValue := c.cache.Get(obj.GetResponseCacheKey()) if cacheValue != nil { @@ -41,6 +45,7 @@ func (c *MemoryCache) Get(obj *object.FileObject) (*response.Response, error) { return nil, errors.New("not found") } +// Delete remove given response from cache func (c *MemoryCache) Delete(obj *object.FileObject) error { c.cache.Delete(obj.GetResponseCacheKey()) return nil diff --git a/pkg/cache/memory_test.go b/pkg/cache/memory_test.go new file mode 100644 index 0000000..f29a514 --- /dev/null +++ b/pkg/cache/memory_test.go @@ -0,0 +1,37 @@ +package cache + +import ( + "testing" + "github.com/aldor007/mort/pkg/object" + "github.com/aldor007/mort/pkg/response" + "github.com/stretchr/testify/assert" +) + +func TestMemoryCache_Set(t *testing.T) { + i := NewMemoryCache(1) + + obj := object.FileObject{} + obj.Key = "cacheKey" + res := response.NewString(200, "test") + + i.Set(&obj, res) + resCache, err := i.Get(&obj) + b, _ := resCache.ReadBody() + assert.Nil(t, err) + + assert.Equal(t, resCache.StatusCode, res.StatusCode) + assert.Equal(t, string(b), "test") +} + +func TestMemoryCache_Delete(t *testing.T) { + i := NewMemoryCache(2) + + obj := object.FileObject{} + obj.Key = "cacheKey" + res := response.NewString(200, "test") + + i.Set(&obj, res) + i.Delete(&obj) + _, err := i.Get(&obj) + assert.NotNil(t, err) +} diff --git a/pkg/cache/redis.go b/pkg/cache/redis.go index fa4e3a9..da5c176 100644 --- a/pkg/cache/redis.go +++ b/pkg/cache/redis.go @@ -22,10 +22,12 @@ func parseAddress(addrs []string) map[string]string { return mp } +// RedisCache store response in redis type RedisCache struct { client *redisCache.Codec } +// NewRedis create connection to redis and update it config from clientConfig map func NewRedis(redisAddress []string, clientConfig map[string]string) *RedisCache { ring := goRedis.NewRing(&goRedis.RingOptions{ Addrs: parseAddress(redisAddress), @@ -50,6 +52,7 @@ func NewRedis(redisAddress []string, clientConfig map[string]string) *RedisCache return &RedisCache{&cache} } +// Set put response into cache func (c *RedisCache) Set(obj *object.FileObject, res *response.Response) error { monitoring.Report().Inc("cache_ratio;status:set") @@ -61,6 +64,7 @@ func (c *RedisCache) Set(obj *object.FileObject, res *response.Response) error { return c.client.Set(&item) } +// Get returns response from cache or error func (c *RedisCache) Get(obj *object.FileObject) (*response.Response, error) { var res response.Response err := c.client.Get(obj.GetResponseCacheKey(), &res) @@ -73,6 +77,7 @@ func (c *RedisCache) Get(obj *object.FileObject) (*response.Response, error) { return &res, err } +// Delete remove response from cache func (c *RedisCache) Delete(obj *object.FileObject) error { return c.client.Delete(obj.GetResponseCacheKey()) } diff --git a/pkg/config/config.go b/pkg/config/config.go index a6005ee..617c0c9 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -170,7 +170,7 @@ func (c *Config) validateStorage(bucketName string, storages StorageTypes) error return err } -func (c *Config) validateTransform(bucketName string, bucket Bucket) error { +func (c *Config) validateTransform(bucketName string, bucket *Bucket) error { transform := bucket.Transform var err error errorMsgPrefix := fmt.Sprintf("%s has invalid transform config", bucketName) @@ -207,6 +207,10 @@ func (c *Config) validateTransform(bucketName string, bucket Bucket) error { } } + if transform.ResultKey == "" && (transform.Kind == "query" || transform.Kind == "presets-query") { + bucket.Transform.ResultKey = "hashParent" + } + return err } @@ -251,7 +255,7 @@ func (c *Config) validateServer() error { } if c.Server.Cache.MaxCacheItemSize == 0 { - c.Server.Cache.MaxCacheItemSize = 50 * 2 << 20 + c.Server.Cache.MaxCacheItemSize = 5 * 2 << 20 } else { c.Server.Cache.MaxCacheItemSize = c.Server.Cache.MaxCacheItemSize * 2 << 20 } @@ -281,7 +285,7 @@ func (c *Config) validate() error { } if bucket.Transform != nil { - err = c.validateTransform(name, bucket) + err = c.validateTransform(name, &bucket) if err != nil { return err } diff --git a/pkg/object/file_object.go b/pkg/object/file_object.go index 345aa7f..8cde538 100644 --- a/pkg/object/file_object.go +++ b/pkg/object/file_object.go @@ -88,6 +88,15 @@ func (o *FileObject) HasTransform() bool { return o.Transforms.NotEmpty == true } +// Type returns type of object "parent" or "transform" +func (o* FileObject) Type() string { + if o.HasTransform(){ + return "transform" + } + + return "parent" +} + // UpdateKey add string to ky func (o *FileObject) UpdateKey(str string) { o.key = o.key + str diff --git a/pkg/object/file_object_test.go b/pkg/object/file_object_test.go index b2a9953..78ca2da 100644 --- a/pkg/object/file_object_test.go +++ b/pkg/object/file_object_test.go @@ -379,6 +379,7 @@ func TestNewFileObjectPresetQueryWatermark(t *testing.T) { parent := obj.Parent assert.Equal(t, "/parent.jpg", parent.Key, "invalid parent key") + assert.Equal(t, "/parent.jpg/90e8c676de60865efa843590826a08e1", obj.Key, "invalid object key") _, err = obj.Transforms.BimgOptions(imageInfo) diff --git a/pkg/object/query.go b/pkg/object/query.go index ae722b3..988f2b4 100644 --- a/pkg/object/query.go +++ b/pkg/object/query.go @@ -106,6 +106,15 @@ func parseOperation(query url.Values) (transforms.Transforms, error) { if err != nil { return trans, err } + case "resizeCropAuto": + var w, h int + w, _ = queryToInt(query, "width") + h, _ = queryToInt(query, "height") + + err = trans.ResizeCropAuto(w, h) + if err != nil { + return trans, err + } case "extract": var w, h, t, l int w, _ = queryToInt(query, "areaWith") diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 4cd84c8..79d8660 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -24,6 +24,7 @@ import ( "strings" "sync" "time" + "fmt" ) const notFound = "{\"error\":\"item not found\"}" @@ -62,6 +63,7 @@ var storageCacheLock = sync.RWMutex{} // Get retrieve obj from given storage and returns its wrapped in response func Get(obj *object.FileObject) *response.Response { + inc(obj, "get") metric := "storage_time;method:get,storage:" + obj.Storage.Kind t := monitoring.Report().Timer(metric) defer t.Done() @@ -114,6 +116,7 @@ func Get(obj *object.FileObject) *response.Response { // Head retrieve obj from given storage and returns its wrapped in response (but only headers, content of object is omitted) func Head(obj *object.FileObject) *response.Response { + inc(obj, "head") metric := "storage_time;method:head,storage:" + obj.Storage.Kind t := monitoring.Report().Timer(metric) defer t.Done() @@ -142,6 +145,7 @@ func Head(obj *object.FileObject) *response.Response { // Set create object on storage wit given body and headers func Set(obj *object.FileObject, metaHeaders http.Header, contentLen int64, body io.Reader) *response.Response { + inc(obj, "set") metric := "storage_time;method:set,storage:" + obj.Storage.Kind t := monitoring.Report().Timer(metric) defer t.Done() @@ -183,6 +187,7 @@ func Set(obj *object.FileObject, metaHeaders http.Header, contentLen int64, body // Delete remove object from given storage func Delete(obj *object.FileObject) *response.Response { + inc(obj, "delete") metric := "storage_time;method:delete,storage:" + obj.Storage.Kind t := monitoring.Report().Timer(metric) defer t.Done() @@ -542,6 +547,11 @@ func parseMetadata(obj *object.FileObject, metadata map[string]interface{}, res } +func inc(obj *object.FileObject, method string) { + monitoring.Report().Inc(fmt.Sprintf("storage_request;method:%s,storage:%s,bucket:%s,object_type:%s", + method, obj.Storage.Kind, obj.Storage.Bucket, obj.Type())) +} + func createBytesHeader(bytesRage string, size int64) string { buf := bufPool.Get().(*bytes.Buffer) defer bufPool.Put(buf) diff --git a/tests-int/Images.Spec.js b/tests-int/Images.Spec.js index 70fe964..a1fc9eb 100644 --- a/tests-int/Images.Spec.js +++ b/tests-int/Images.Spec.js @@ -5,6 +5,37 @@ const expect = chai.expect; const host = 'localhost:' + process.env.MORT_PORT; const request = supertest(`http://${host}`); +function checkImage(reqPath, width, height, sizeRange, done) { + request.get(reqPath) + .expect(200) + .end(function(err, res) { + if (err) { + return done(err); + } + + const body = res.body; + expect(body.length).to.be.within(...sizeRange); + + expect(res.headers['x-amz-meta-public-width']).to.eql(width); + expect(res.headers['x-amz-meta-public-height']).to.eql(height); + request.get(reqPath) + .expect(200) + .end(function (err2, res2) { + if (err2) { + return done(err2); + } + + + expect(res2.headers['x-amz-meta-public-width']).to.eql(width); + expect(res2.headers['x-amz-meta-public-height']).to.eql(height); + + const body2 = res2.body; + expect(body2.length).to.eql(body.length); + done(err2) + }); + }); +} + describe('Image processing', function () { describe('presets', function () { @@ -46,34 +77,7 @@ describe('Image processing', function () { const reqPath = '/remote/nxpvwo7qqfwz.jpg/crop'; const width = '756'; const height = '396'; - request.get(reqPath) - .expect(200) - .end(function(err, res) { - if (err) { - return done(err); - } - - const body = res.body; - expect(body.length).to.be.within(30950, 33000); - - expect(res.headers['x-amz-meta-public-width']).to.eql(width); - expect(res.headers['x-amz-meta-public-height']).to.eql(height); - request.get(reqPath) - .expect(200) - .end(function (err2, res2) { - if (err2) { - return done(err2); - } - - - expect(res2.headers['x-amz-meta-public-width']).to.eql(width); - expect(res2.headers['x-amz-meta-public-height']).to.eql(height); - - const body2 = res2.body; - expect(body2.length).to.eql(body.length); - done(err2) - }); - }); + checkImage(reqPath, width, height, [30950, 33000], done) }); it('should create thumbnails with blur from external source', function (done) { @@ -81,33 +85,7 @@ describe('Image processing', function () { const reqPath = '/remote/nxpvwo7qqfwz.jpg/blur'; const width = '756'; const height = '396'; - request.get(reqPath) - .expect(200) - .end(function(err, res) { - if (err) { - return done(err); - } - - const body = res.body; - expect(body.length).to.be.within(10222, 11000); - - expect(res.headers['x-amz-meta-public-width']).to.eql(width); - expect(res.headers['x-amz-meta-public-height']).to.eql(height); - request.get(reqPath) - .expect(200) - .end(function (err2, res2) { - if (err2) { - return done(err2); - } - - const body2 = res2.body; - expect(body2.length).to.eql(body.length); - - expect(res2.headers['x-amz-meta-public-width']).to.eql(width); - expect(res2.headers['x-amz-meta-public-height']).to.eql(height); - done(err2) - }); - }); + checkImage(reqPath, width, height, [10222, 11000], done); }); it('should create thumbnails with watermark from external source', function (done) { @@ -115,33 +93,15 @@ describe('Image processing', function () { const reqPath = '/remote/nxpvwo7qqfwz.jpg/watermark'; const width = '200'; const height = '200'; - request.get(reqPath) - .expect(200) - .end(function(err, res) { - if (err) { - return done(err); - } - - const body = res.body; - expect(body.length).to.be.within(4500, 4600); - - expect(res.headers['x-amz-meta-public-width']).to.eql(width); - expect(res.headers['x-amz-meta-public-height']).to.eql(height); - request.get(reqPath) - .expect(200) - .end(function (err2, res2) { - if (err2) { - return done(err2); - } - - const body2 = res2.body; - expect(body2.length).to.eql(body.length); + checkImage(reqPath, width, height, [4500, 4600], done) + }); - expect(res2.headers['x-amz-meta-public-width']).to.eql(width); - expect(res2.headers['x-amz-meta-public-height']).to.eql(height); - done(err2) - }); - }); + it('should extract given area from image', function (done) { + this.timeout(5000); + const reqPath = '/remote/nxpvwo7qqfwz.jpg/extract'; + const width = '700'; + const height = '496'; + checkImage(reqPath, width, height, [60000, 70855], done) }); it('should create thumbnails with watermark from external source and handle accept header', function (done) { @@ -176,6 +136,7 @@ describe('Image processing', function () { expect(res2.headers['x-amz-meta-public-width']).to.eql(width); expect(res2.headers['x-amz-meta-public-height']).to.eql(height); + expect(res2.headers['content-type']).to.eql('image/webp'); done(err2) }); }); @@ -244,7 +205,7 @@ describe('Image processing', function () { } const body = res.body; - expect(body.length).to.be.within(1600, 2054); + expect(body.length).to.be.within(1500, 2054); expect(res.headers['x-amz-meta-public-width']).to.eql('100'); expect(res.headers['x-amz-meta-public-height']).to.eql('100'); @@ -279,7 +240,7 @@ describe('Image processing', function () { const body = res.body; const width = '400'; const height = '100'; - expect(body.length).to.be.within(1160, 2000); + expect(body.length).to.be.within(1060, 2000); expect(res.headers['x-amz-meta-public-width']).to.eql(width); expect(res.headers['x-amz-meta-public-height']).to.eql(height); @@ -303,69 +264,17 @@ describe('Image processing', function () { it('should create thumbnails and rotate', function (done) { this.timeout(5000); const reqPath = '/remote/nxpvwo7qqfwz.jpg?operation=resize&width=400&operation=rotate&angle=90'; - request.get(reqPath) - .expect(200) - .end(function(err, res) { - if (err) { - return done(err); - } - - const body = res.body; - const width = '400'; - const height = '320'; - expect(body.length).to.be.within(18100, 18300); - - expect(res.headers['x-amz-meta-public-width']).to.eql(width); - expect(res.headers['x-amz-meta-public-height']).to.eql(height); - request.get(reqPath) - .expect(200) - .end(function (err2, res2) { - if (err2) { - return done(err2); - } - - const body2 = res2.body; - expect(body2.length).to.eql(body.length); - - expect(res2.headers['x-amz-meta-public-width']).to.eql(width); - expect(res2.headers['x-amz-meta-public-height']).to.eql(height); - done(err2) - }); - }); + const width = '400'; + const height = '320'; + checkImage(reqPath, width, height, [18100, 18300], done); }); it('should create thumbnails with watermark', function (done) { this.timeout(5000); const reqPath = '/remote/nxpvwo7qqfwz.jpg?operation=resize&width=400&height=100&image=https://i.imgur.com/uomkVIL.png&opacity=0.5&position=top-left&operation=watermark'; - request.get(reqPath) - .expect(200) - .end(function(err, res) { - if (err) { - return done(err); - } - - const body = res.body; - const width = '400'; - const height = '100'; - expect(body.length).to.be.within(4000, 4600); - - expect(res.headers['x-amz-meta-public-width']).to.eql(width); - expect(res.headers['x-amz-meta-public-height']).to.eql(height); - request.get(reqPath) - .expect(200) - .end(function (err2, res2) { - if (err2) { - return done(err2); - } - - const body2 = res2.body; - expect(body2.length).to.eql(body.length); - - expect(res2.headers['x-amz-meta-public-width']).to.eql(width); - expect(res2.headers['x-amz-meta-public-height']).to.eql(height); - done(err2) - }); - }); + const width = '400'; + const height = '100'; + checkImage(reqPath, width, height, [4000, 4600], done); }); it('should return 404 when parent not found', function (done) { diff --git a/tests-int/config.yml b/tests-int/config.yml index 6c5ed76..6419fb2 100644 --- a/tests-int/config.yml +++ b/tests-int/config.yml @@ -88,6 +88,12 @@ buckets: width: 756 height: 396 mode: outbound + extract: + quality: 75 + filters: + extract: + width: 700 + height: 496 blur: quality: 75 filters: