Thumbnails Service

changelog/unreleased/thumbnail-service.md

@@ -0,0 +1,6 @@
+Change: implement the first working version
+
+We implemented the first simple version.
+It can load images via webdav and store them locally in the filesystem.
+
+https://github.com/owncloud/ocis-thumbnails/pull/3

go.mod

@@ -6,16 +6,19 @@ require (
 	contrib.go.opencensus.io/exporter/jaeger v0.2.0
 	contrib.go.opencensus.io/exporter/ocagent v0.6.0
 	contrib.go.opencensus.io/exporter/zipkin v0.1.1
-	github.com/UnnoTed/fileb0x v1.1.4 // indirect
-	github.com/cespare/reflex v0.2.0 // indirect
-	github.com/go-chi/chi v4.0.2+incompatible
-	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
+	github.com/UnnoTed/fileb0x v1.1.4
+	github.com/cespare/reflex v0.2.0
+	github.com/golang/protobuf v1.3.2
+	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
 	github.com/micro/cli/v2 v2.1.1
-	github.com/ogier/pflag v0.0.1 // indirect
+	github.com/micro/go-micro/v2 v2.0.0
+	github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
+	github.com/ogier/pflag v0.0.1
 	github.com/oklog/run v1.0.0
 	github.com/openzipkin/zipkin-go v0.2.2
 	github.com/owncloud/ocis-pkg/v2 v2.0.1
-	github.com/restic/calens v0.2.0 // indirect
+	github.com/prometheus/client_golang v1.2.1
+	github.com/restic/calens v0.2.0
 	github.com/spf13/viper v1.5.0
 	go.opencensus.io v0.22.2
 )

pkg/command/server.go

@@ -2,9 +2,9 @@ package command
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"os/signal"
-	"strings"
 	"time"
 
 	"contrib.go.opencensus.io/exporter/jaeger"
@@ -17,8 +17,7 @@ import (
 	"github.com/owncloud/ocis-thumbnails/pkg/config"
 	"github.com/owncloud/ocis-thumbnails/pkg/flagset"
 	"github.com/owncloud/ocis-thumbnails/pkg/metrics"
-	"github.com/owncloud/ocis-thumbnails/pkg/server/debug"
-	"github.com/owncloud/ocis-thumbnails/pkg/server/http"
+	"github.com/owncloud/ocis-thumbnails/pkg/server/grpc"
 	"go.opencensus.io/stats/view"
 	"go.opencensus.io/trace"
 )
@@ -30,10 +29,6 @@ func Server(cfg *config.Config) *cli.Command {
 		Usage: "Start integrated server",
 		Flags: flagset.ServerWithConfig(cfg),
 		Before: func(c *cli.Context) error {
-			if cfg.HTTP.Root != "/" {
-				cfg.HTTP.Root = strings.TrimSuffix(cfg.HTTP.Root, "/")
-			}
-
 			return nil
 		},
 		Action: func(c *cli.Context) error {
@@ -131,72 +126,22 @@ func Server(cfg *config.Config) *cli.Command {
 
 			defer cancel()
 
-			{
-				server, err := http.Server(
-					http.Logger(logger),
-					http.Context(ctx),
-					http.Config(cfg),
-					http.Metrics(metrics),
-					http.Flags(flagset.RootWithConfig(cfg)),
-					http.Flags(flagset.ServerWithConfig(cfg)),
-				)
-
-				if err != nil {
-					logger.Info().
-						Err(err).
-						Str("transport", "http").
-						Msg("Failed to initialize server")
-
-					return err
-				}
-
-				gr.Add(func() error {
-					return server.Run()
-				}, func(_ error) {
-					logger.Info().
-						Str("transport", "http").
-						Msg("Shutting down server")
-
-					cancel()
-				})
-			}
-
-			{
-				server, err := debug.Server(
-					debug.Logger(logger),
-					debug.Context(ctx),
-					debug.Config(cfg),
-				)
-
-				if err != nil {
-					logger.Info().
-						Err(err).
-						Str("transport", "debug").
-						Msg("Failed to initialize server")
-
-					return err
-				}
-
-				gr.Add(func() error {
-					return server.ListenAndServe()
-				}, func(_ error) {
-					ctx, timeout := context.WithTimeout(ctx, 5*time.Second)
-
-					defer timeout()
-					defer cancel()
+			service := grpc.NewService(
+				grpc.Logger(logger),
+				grpc.Context(ctx),
+				grpc.Config(cfg),
+				grpc.Name(cfg.Server.Name),
+				grpc.Namespace(cfg.Server.Namespace),
+				grpc.Address(cfg.Server.Address),
+				grpc.Metrics(metrics),
+			)
 
-					if err := server.Shutdown(ctx); err != nil {
-						logger.Info().
-							Err(err).
-							Str("transport", "debug").
-							Msg("Failed to shutdown server")
-					} else {
-						logger.Info().
-							Str("transport", "debug").
-							Msg("Shutting down server")
-					}
-				})
-			}
+			gr.Add(func() error {
+				return service.Run()
+			}, func(_ error) {
+				fmt.Println("shutting down grpc server")
+				cancel()
+			})
 
 			{
 				stop := make(chan os.Signal, 1)

pkg/config/config.go

@@ -15,11 +15,11 @@ type Debug struct {
 	Zpages bool
 }
 
-// HTTP defines the available http configuration.
-type HTTP struct {
-	Addr      string
+// Server defines the available server configuration.
+type Server struct {
+	Name      string
 	Namespace string
-	Root      string
+	Address   string
 }
 
 // Tracing defines the available tracing configuration.
@@ -33,11 +33,23 @@ type Tracing struct {
 
 // Config combines all available configuration parts.
 type Config struct {
-	File    string
-	Log     Log
-	Debug   Debug
-	HTTP    HTTP
-	Tracing Tracing
+	File              string
+	Log               Log
+	Debug             Debug
+	Server            Server
+	Tracing           Tracing
+	FileSystemStorage FileSystemStorage
+	WebDavSource      WebDavSource
+}
+
+// FileSystemStorage defines the available filesystem storage configuration.
+type FileSystemStorage struct {
+	RootDirectory string
+}
+
+// WebDavSource defines the available webdav source configuration.
+type WebDavSource struct {
+	BaseURL string
 }
 
 // New initializes a new configuration with or without defaults.

pkg/flagset/flagset.go

@@ -1,6 +1,9 @@
 package flagset
 
 import (
+	"os"
+	"path/filepath"
+
 	"github.com/micro/cli/v2"
 	"github.com/owncloud/ocis-thumbnails/pkg/config"
 )
@@ -42,7 +45,7 @@ func HealthWithConfig(cfg *config.Config) []cli.Flag {
 	return []cli.Flag{
 		&cli.StringFlag{
 			Name:        "debug-addr",
-			Value:       "0.0.0.0:9114",
+			Value:       "0.0.0.0:9189",
 			Usage:       "Address to debug endpoint",
 			EnvVars:     []string{"THUMBNAILS_DEBUG_ADDR"},
 			Destination: &cfg.Debug.Addr,
@@ -89,7 +92,7 @@ func ServerWithConfig(cfg *config.Config) []cli.Flag {
 		},
 		&cli.StringFlag{
 			Name:        "debug-addr",
-			Value:       "0.0.0.0:9114",
+			Value:       "0.0.0.0:9189",
 			Usage:       "Address to bind debug server",
 			EnvVars:     []string{"THUMBNAILS_DEBUG_ADDR"},
 			Destination: &cfg.Debug.Addr,
@@ -114,25 +117,39 @@ func ServerWithConfig(cfg *config.Config) []cli.Flag {
 			Destination: &cfg.Debug.Zpages,
 		},
 		&cli.StringFlag{
-			Name:        "http-addr",
-			Value:       "0.0.0.0:9110",
-			Usage:       "Address to bind http server",
-			EnvVars:     []string{"THUMBNAILS_HTTP_ADDR"},
-			Destination: &cfg.HTTP.Addr,
+			Name:        "grpc-name",
+			Value:       "thumbnails",
+			Usage:       "Name of the service",
+			EnvVars:     []string{"THUMBNAILS_GRPC_NAME"},
+			Destination: &cfg.Server.Name,
+		},
+		&cli.StringFlag{
+			Name:        "grpc-addr",
+			Value:       "0.0.0.0:9185",
+			Usage:       "Address to bind grpc server",
+			EnvVars:     []string{"THUMBNAILS_GRPC_ADDR"},
+			Destination: &cfg.Server.Address,
+		},
+		&cli.StringFlag{
+			Name:        "grpc-namespace",
+			Value:       "com.owncloud.api",
+			Usage:       "Set the base namespace for the grpc namespace",
+			EnvVars:     []string{"THUMBNAILS_GRPC_NAMESPACE"},
+			Destination: &cfg.Server.Namespace,
 		},
 		&cli.StringFlag{
-			Name:        "http-namespace",
-			Value:       "com.owncloud.web",
-			Usage:       "Set the base namespace for the http namespace",
-			EnvVars:     []string{"THUMBNAILS_HTTP_NAMESPACE"},
-			Destination: &cfg.HTTP.Namespace,
+			Name:        "filesystemstorage-root",
+			Value:       filepath.Join(os.TempDir(), "ocis-thumbnails/"),
+			Usage:       "Root path of the filesystem storage directory",
+			EnvVars:     []string{"THUMBNAILS_FILESYSTEMSTORAGE_ROOT"},
+			Destination: &cfg.FileSystemStorage.RootDirectory,
 		},
 		&cli.StringFlag{
-			Name:        "http-root",
-			Value:       "/",
-			Usage:       "Root path of http server",
-			EnvVars:     []string{"THUMBNAILS_HTTP_ROOT"},
-			Destination: &cfg.HTTP.Root,
+			Name:        "webdavsource-baseurl",
+			Value:       "http://localhost:9140/remote.php/webdav/",
+			Usage:       "Base url for a webdav api",
+			EnvVars:     []string{"THUMBNAILS_WEBDAVSOURCE_BASEURL"},
+			Destination: &cfg.WebDavSource.BaseURL,
 		},
 	}
 }

pkg/metrics/metrics.go

@@ -1,5 +1,7 @@
 package metrics
 
+import "github.com/prometheus/client_golang/prometheus"
+
 var (
 	// Namespace defines the namespace for the defines metrics.
 	Namespace = "ocis"
@@ -10,23 +12,45 @@ var (
 
 // Metrics defines the available metrics of this service.
 type Metrics struct {
-	// Counter  *prometheus.CounterVec
+	Counter  *prometheus.CounterVec
+	Latency  *prometheus.SummaryVec
+	Duration *prometheus.HistogramVec
 }
 
 // New initializes the available metrics.
 func New() *Metrics {
 	m := &Metrics{
-		// Counter: prometheus.NewCounterVec(prometheus.CounterOpts{
-		// 	Namespace: Namespace,
-		// 	Subsystem: Subsystem,
-		// 	Name:      "greet_total",
-		// 	Help:      "How many greeting requests processed",
-		// }, []string{}),
+		Counter: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Namespace: Namespace,
+			Subsystem: Subsystem,
+			Name:      "getthumbnail_total",
+			Help:      "How many GetThumbnail requests processed",
+		}, []string{}),
+		Latency: prometheus.NewSummaryVec(prometheus.SummaryOpts{
+			Namespace: Namespace,
+			Subsystem: Subsystem,
+			Name:      "getthumbnail_latency_microseconds",
+			Help:      "GetThumbnail request latencies in microseconds",
+		}, []string{}),
+		Duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
+			Namespace: Namespace,
+			Subsystem: Subsystem,
+			Name:      "getthumbnail_duration_seconds",
+			Help:      "GetThumbnail method requests time in seconds",
+		}, []string{}),
 	}
 
-	// prometheus.Register(
-	// 	m.Counter,
-	// )
+	prometheus.Register(
+		m.Counter,
+	)
+
+	prometheus.Register(
+		m.Latency,
+	)
+
+	prometheus.Register(
+		m.Duration,
+	)
 
 	return m
 }