diff --git a/go.mod b/go.mod index 97196b9..3744a37 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jmoiron/sqlx v1.3.5 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect diff --git a/go.sum b/go.sum index 2248fcc..0f7d0a4 100644 --- a/go.sum +++ b/go.sum @@ -69,8 +69,10 @@ github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbS github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -141,8 +143,13 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= +github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -150,6 +157,7 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= @@ -158,10 +166,14 @@ github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZb github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= @@ -522,6 +534,7 @@ gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/server/v1alpha1/handlers.go b/internal/server/v1alpha1/handlers.go index e04ac02..f97f29b 100644 --- a/internal/server/v1alpha1/handlers.go +++ b/internal/server/v1alpha1/handlers.go @@ -87,6 +87,8 @@ func (s *Server) WebhookHandler() http.HandlerFunc { // it will call the security pipeline if configured and store data on each configured // storages func webhookService(s *Server, spec *config.WebhookSpec, r *http.Request) (err error) { + ctx := r.Context() + if spec == nil { return config.ErrSpecNotFound } @@ -107,21 +109,28 @@ func webhookService(s *Server, spec *config.WebhookSpec, r *http.Request) (err e } } + previousPayload := data + payloadFormatter := formatting.New(). + WithRequest(r). + WithPayload(data). + WithData("Spec", spec). + WithData("Config", config.Current()) + for _, storage := range spec.Storage { - str, err := formatting. - NewTemplateData(storage.Formatting.Template). - WithRequest(r). - WithPayload(data). - WithData("Spec", spec). - WithData("Storage", storage). - WithData("Config", config.Current()). - Render() + payloadFormatter = payloadFormatter.WithData("Storage", storage) + + storagePayload, err := payloadFormatter.WithTemplate(storage.Formatting.Template).Render() if err != nil { return err } - log.Debug().Msgf("store following data: %+v", str) - if err := storage.Client.Push(str); err != nil { + // update the formatter with the rendered payload of storage formatting + // this will allow to chain formatting + payloadFormatter.WithData("PreviousPayload", previousPayload) + ctx = formatting.ToContext(ctx, payloadFormatter) + + log.Debug().Msgf("store following data: %s", storagePayload) + if err := storage.Client.Push(ctx, []byte(storagePayload)); err != nil { return err } log.Debug().Str("storage", storage.Client.Name()).Msgf("stored successfully") diff --git a/pkg/formatting/format.go b/pkg/formatting/format.go deleted file mode 100644 index 0bd9b17..0000000 --- a/pkg/formatting/format.go +++ /dev/null @@ -1,78 +0,0 @@ -package formatting - -import ( - "bytes" - "fmt" - "net/http" - "sync" - "text/template" - - "github.com/rs/zerolog/log" -) - -type TemplateData struct { - tmplString string - - mu sync.RWMutex // protect following field amd template parsing - data map[string]interface{} -} - -// NewTemplateData returns a new TemplateData instance. It takes the template -// string as a parameter. The template string is the string that will be used -// to render the template. The data is the map of data that will be used to -// render the template. -func NewTemplateData(tmplString string) *TemplateData { - return &TemplateData{ - tmplString: tmplString, - data: make(map[string]interface{}), - mu: sync.RWMutex{}, - } -} - -// WithData adds a key-value pair to the data map. The key is the name of the -// variable and the value is the value of the variable. -func (d *TemplateData) WithData(name string, data interface{}) *TemplateData { - d.mu.Lock() - defer d.mu.Unlock() - - d.data[name] = data - return d -} - -// WithRequest adds a http.Request object to the data map. The key of request is -// "Request". -func (d *TemplateData) WithRequest(r *http.Request) *TemplateData { - d.WithData("Request", r) - return d -} - -// WithPayload adds a payload to the data map. The key of payload is "Payload". -// The payload is basically the body of the request. -func (d *TemplateData) WithPayload(payload []byte) *TemplateData { - d.WithData("Payload", string(payload)) - return d -} - -// Render returns the rendered template string. It takes the template string -// from the TemplateData instance and the data stored in the TemplateData -// instance. It returns an error if the template string is invalid or when -// rendering the template fails. -func (d *TemplateData) Render() (string, error) { - d.mu.RLock() - defer d.mu.RUnlock() - - log.Debug().Msgf("rendering template: %s", d.tmplString) - - t := template.New("formattingTmpl").Funcs(funcMap()) - t, err := t.Parse(d.tmplString) - if err != nil { - return "", fmt.Errorf("error in your template: %s", err.Error()) - } - - buf := new(bytes.Buffer) - if err := t.Execute(buf, d.data); err != nil { - return "", fmt.Errorf("error while filling your template: %s", err.Error()) - } - - return buf.String(), nil -} diff --git a/pkg/formatting/formatter.go b/pkg/formatting/formatter.go new file mode 100644 index 0000000..d32dd10 --- /dev/null +++ b/pkg/formatting/formatter.go @@ -0,0 +1,128 @@ +package formatting + +import ( + "bytes" + "context" + "fmt" + "net/http" + "sync" + "text/template" + + "github.com/rs/zerolog/log" +) + +type Formatter struct { + tmplString string + + mu sync.RWMutex // protect following field amd template parsing + data map[string]interface{} +} + +var ( + formatterCtxKey = struct{}{} + // ErrNotFoundInContext is returned when the formatting data is not found in + // the context. Use `FromContext` and `ToContext` to set and get the data in + // the context. + ErrNotFoundInContext = fmt.Errorf("unable to get the formatting data from the context") + // ErrNoTemplate is returned when no template is defined in the Formatter + // instance. Provide a template using the WithTemplate method. + ErrNoTemplate = fmt.Errorf("no template defined") +) + +// NewWithTemplate returns a new Formatter instance. It takes the template +// string as a parameter. The template string is the string that will be used +// to render the template. The data is the map of data that will be used to +// render the template. +// ! DEPRECATED: use New() and WithTemplate() instead +func NewWithTemplate(tmplString string) *Formatter { + return &Formatter{ + tmplString: tmplString, + data: make(map[string]interface{}), + mu: sync.RWMutex{}, + } +} + +// New returns a new Formatter instance. It takes no parameters. The template +// string must be set using the WithTemplate method. The data is the map of data +// that will be used to render the template. +func New() *Formatter { + return &Formatter{ + data: make(map[string]interface{}), + mu: sync.RWMutex{}, + } +} + +// WithTemplate sets the template string. The template string is the string that +// will be used to render the template. +func (d *Formatter) WithTemplate(tmplString string) *Formatter { + d.tmplString = tmplString + return d +} + +// WithData adds a key-value pair to the data map. The key is the name of the +// variable and the value is the value of the variable. +func (d *Formatter) WithData(name string, data interface{}) *Formatter { + d.mu.Lock() + defer d.mu.Unlock() + + d.data[name] = data + return d +} + +// WithRequest adds a http.Request object to the data map. The key of request is +// "Request". +func (d *Formatter) WithRequest(r *http.Request) *Formatter { + d.WithData("Request", r) + return d +} + +// WithPayload adds a payload to the data map. The key of payload is "Payload". +// The payload is basically the body of the request. +func (d *Formatter) WithPayload(payload []byte) *Formatter { + d.WithData("Payload", string(payload)) + return d +} + +// Render returns the rendered template string. It takes the template string +// from the Formatter instance and the data stored in the Formatter +// instance. It returns an error if the template string is invalid or when +// rendering the template fails. +func (d *Formatter) Render() (string, error) { + d.mu.RLock() + defer d.mu.RUnlock() + + if d.tmplString == "" { + return "", ErrNoTemplate + } + + log.Debug().Msgf("rendering template: %s", d.tmplString) + + t := template.New("formattingTmpl").Funcs(funcMap()) + t, err := t.Parse(d.tmplString) + if err != nil { + return "", fmt.Errorf("error in your template: %s", err.Error()) + } + + buf := new(bytes.Buffer) + if err := t.Execute(buf, d.data); err != nil { + return "", fmt.Errorf("error while filling your template: %s", err.Error()) + } + + return buf.String(), nil +} + +// FromContext returns the Formatter instance stored in the context. It returns +// an error if the Formatter instance is not found in the context. +func FromContext(ctx context.Context) (*Formatter, error) { + d, ok := ctx.Value(formatterCtxKey).(*Formatter) + if !ok { + return nil, ErrNotFoundInContext + } + return d, nil +} + +// ToContext adds the Formatter instance to the context. It returns the context +// with the Formatter instance. +func ToContext(ctx context.Context, d *Formatter) context.Context { + return context.WithValue(ctx, formatterCtxKey, d) +} diff --git a/pkg/formatting/format_test.go b/pkg/formatting/formatter_test.go similarity index 61% rename from pkg/formatting/format_test.go rename to pkg/formatting/formatter_test.go index f4efe95..da11a6f 100644 --- a/pkg/formatting/format_test.go +++ b/pkg/formatting/formatter_test.go @@ -1,25 +1,29 @@ package formatting import ( + "context" "encoding/json" "net/http" "net/http/httptest" "testing" "github.com/stretchr/testify/assert" - - "atomys.codes/webhooked/internal/config" ) -func TestNewTemplateData(t *testing.T) { +func TestNewWithTemplate(t *testing.T) { assert := assert.New(t) - tmpl := NewTemplateData("") + tmpl := New().WithTemplate("") assert.NotNil(tmpl) assert.Equal("", tmpl.tmplString) assert.Equal(0, len(tmpl.data)) - tmpl = NewTemplateData("{{ .Payload }}") + tmpl = New().WithTemplate("{{ .Payload }}") + assert.NotNil(tmpl) + assert.Equal("{{ .Payload }}", tmpl.tmplString) + assert.Equal(0, len(tmpl.data)) + + tmpl = NewWithTemplate("{{ .Payload }}") assert.NotNil(tmpl) assert.Equal("{{ .Payload }}", tmpl.tmplString) assert.Equal(0, len(tmpl.data)) @@ -28,7 +32,7 @@ func TestNewTemplateData(t *testing.T) { func Test_WithData(t *testing.T) { assert := assert.New(t) - tmpl := NewTemplateData("").WithData("test", true) + tmpl := New().WithTemplate("").WithData("test", true) assert.NotNil(tmpl) assert.Equal("", tmpl.tmplString) assert.Equal(1, len(tmpl.data)) @@ -38,7 +42,7 @@ func Test_WithData(t *testing.T) { func Test_WithRequest(t *testing.T) { assert := assert.New(t) - tmpl := NewTemplateData("").WithRequest(httptest.NewRequest("GET", "/", nil)) + tmpl := New().WithTemplate("").WithRequest(httptest.NewRequest("GET", "/", nil)) assert.NotNil(tmpl) assert.Equal("", tmpl.tmplString) assert.Equal(1, len(tmpl.data)) @@ -53,7 +57,7 @@ func Test_WithPayload(t *testing.T) { data, err := json.Marshal(map[string]interface{}{"test": "test"}) assert.Nil(err) - tmpl := NewTemplateData("").WithPayload(data) + tmpl := New().WithTemplate("").WithPayload(data) assert.NotNil(tmpl) assert.Equal("", tmpl.tmplString) assert.Equal(1, len(tmpl.data)) @@ -63,8 +67,12 @@ func Test_WithPayload(t *testing.T) { func Test_Render(t *testing.T) { assert := assert.New(t) + // Test with no template + _, err := New().Render() + assert.ErrorIs(err, ErrNoTemplate) + // Test with basic template - tmpl := NewTemplateData("{{ .Payload }}").WithPayload([]byte(`{"test": "test"}`)) + tmpl := New().WithTemplate("{{ .Payload }}").WithPayload([]byte(`{"test": "test"}`)) assert.NotNil(tmpl) assert.Equal("{{ .Payload }}", tmpl.tmplString) assert.Equal(1, len(tmpl.data)) @@ -79,11 +87,9 @@ func Test_Render(t *testing.T) { req := httptest.NewRequest("GET", "/", nil) req.Header.Set("X-Test", "test") - tmpl = NewTemplateData(` + tmpl = New().WithTemplate(` { - "config": {{ toJson .Config }}, - "spec": {{ toJson .Spec }}, - "storage": {{ toJson .Storage }}, + "customData": {{ toJson .CustomData }}, "metadata": { "testID": "{{ .Request.Header | getHeader "X-Test" }}", "deliveryID": "{{ .Request.Header | getHeader "X-Delivery" | default "unknown" }}" @@ -97,27 +103,14 @@ func Test_Render(t *testing.T) { `). WithPayload([]byte(`{"test": {"foo": true}}`)). WithRequest(req). - WithData("Spec", &config.WebhookSpec{Name: "test", EntrypointURL: "/webhooks/test", Formatting: &config.FormattingSpec{}}). - WithData("Storage", &config.StorageSpec{Type: "testing", Specs: map[string]interface{}{}}). - WithData("Config", config.Current()) + WithData("CustomData", map[string]string{"foo": "bar"}) assert.NotNil(tmpl) str, err = tmpl.Render() assert.Nil(err) assert.JSONEq(`{ - "config": { - "apiVersion":"", - "observability":{ - "metricsEnabled":false - }, - "specs": null - }, - "spec": { - "name":"test", - "entrypointUrl": "/webhooks/test" - }, - "storage": { - "type": "testing" + "customData": { + "foo": "bar" }, "metadata": { "testID": "test", @@ -129,7 +122,7 @@ func Test_Render(t *testing.T) { }`, str) // Test with template with template error - tmpl = NewTemplateData("{{ .Payload }") + tmpl = New().WithTemplate("{{ .Payload }") assert.NotNil(tmpl) assert.Equal("{{ .Payload }", tmpl.tmplString) @@ -139,7 +132,7 @@ func Test_Render(t *testing.T) { assert.Equal("", str) // Test with template with data error - tmpl = NewTemplateData("{{ .Request.Method }}").WithRequest(nil) + tmpl = New().WithTemplate("{{ .Request.Method }}").WithRequest(nil) assert.NotNil(tmpl) assert.Equal("{{ .Request.Method }}", tmpl.tmplString) @@ -148,3 +141,29 @@ func Test_Render(t *testing.T) { assert.Contains(err.Error(), "error while filling your template: ") assert.Equal("", str) } + +func TestFromContext(t *testing.T) { + // Test case 1: context value is not a *Formatter + ctx1 := context.Background() + _, err1 := FromContext(ctx1) + assert.Equal(t, ErrNotFoundInContext, err1) + + // Test case 2: context value is a *Formatter + ctx2 := context.WithValue(context.Background(), formatterCtxKey, &Formatter{}) + formatter, err2 := FromContext(ctx2) + assert.NotNil(t, formatter) + assert.Nil(t, err2) +} + +func TestToContext(t *testing.T) { + // Test case 1: context value is nil + ctx1 := context.Background() + ctx1 = ToContext(ctx1, nil) + assert.Nil(t, ctx1.Value(formatterCtxKey)) + + // Test case 2: context value is not nil + ctx2 := context.Background() + formatter := &Formatter{} + ctx2 = ToContext(ctx2, formatter) + assert.Equal(t, formatter, ctx2.Value(formatterCtxKey)) +} diff --git a/pkg/storage/postgres/postgres.go b/pkg/storage/postgres/postgres.go index 39b62fc..9429fcb 100644 --- a/pkg/storage/postgres/postgres.go +++ b/pkg/storage/postgres/postgres.go @@ -1,18 +1,21 @@ package postgres import ( - "database/sql" + "context" "fmt" + "github.com/jmoiron/sqlx" _ "github.com/lib/pq" + "github.com/rs/zerolog/log" "atomys.codes/webhooked/internal/valuable" + "atomys.codes/webhooked/pkg/formatting" ) // storage is the struct contains client and config // Run is made from external caller at begins programs type storage struct { - client *sql.DB + client *sqlx.DB config *config } @@ -20,8 +23,16 @@ type storage struct { // Run is made from internal caller type config struct { DatabaseURL valuable.Valuable `mapstructure:"databaseUrl" json:"databaseUrl"` - TableName string `mapstructure:"tableName" json:"tableName"` - DataField string `mapstructure:"dataField" json:"dataField"` + // ! Deprecation notice: End of life in v1.0.0 + TableName string `mapstructure:"tableName" json:"tableName"` + // ! Deprecation notice: End of life in v1.0.0 + DataField string `mapstructure:"dataField" json:"dataField"` + + UseFormattingToPerformQuery bool `mapstructure:"useFormattingToPerformQuery" json:"useFormattingToPerformQuery"` + // The query to perform on the database with named arguments + Query string `mapstructure:"query" json:"query"` + // The arguments to use in the query with the formatting feature (see pkg/formatting) + Args map[string]string `mapstructure:"args" json:"args"` } // NewStorage is the function for create new Postgres client storage @@ -40,7 +51,26 @@ func NewStorage(configRaw map[string]interface{}) (*storage, error) { return nil, err } - if newClient.client, err = sql.Open("postgres", newClient.config.DatabaseURL.First()); err != nil { + // ! Deprecation notice: End of life in v1.0.0 + if newClient.config.TableName != "" || newClient.config.DataField != "" { + log.Warn().Msg("[DEPRECATION NOTICE] The TableName and DataField are deprecated, please use the formatting feature instead") + } + + if newClient.config.UseFormattingToPerformQuery { + if newClient.config.TableName != "" || newClient.config.DataField != "" { + return nil, fmt.Errorf("the formatting feature is enabled, the TableName and DataField are deprecated and cannot be used in the same time") + } + + if newClient.config.Query == "" { + return nil, fmt.Errorf("the query is required when the formatting feature is enabled") + } + + if newClient.config.Args == nil { + newClient.config.Args = make(map[string]string, 0) + } + } + + if newClient.client, err = sqlx.Open("postgres", newClient.config.DatabaseURL.First()); err != nil { return nil, err } @@ -53,15 +83,46 @@ func (c storage) Name() string { return "postgres" } -// Push is the function for push data in the storage +// Push is the function for push data in the storage. +// The data is formatted with the formatting feature and be serialized by the +// client with "toSql" method // A run is made from external caller // @param value that will be pushed // @return an error if the push failed -func (c storage) Push(value interface{}) error { - request := fmt.Sprintf("INSERT INTO %s(%s) VALUES ($1)", c.config.TableName, c.config.DataField) - if _, err := c.client.Query(request, value); err != nil { +func (c storage) Push(ctx context.Context, value []byte) error { + // ! Deprecation notice: End of life in v1.0.0 + if !c.config.UseFormattingToPerformQuery { + request := fmt.Sprintf("INSERT INTO %s(%s) VALUES ($1)", c.config.TableName, c.config.DataField) + if _, err := c.client.Query(request, value); err != nil { + return err + } + return nil + } + + formatter, err := formatting.FromContext(ctx) + if err != nil { + return err + } + + stmt, err := c.client.PrepareNamedContext(ctx, c.config.Query) + if err != nil { return err } - return nil + var namedArgs = make(map[string]interface{}, 0) + for name, template := range c.config.Args { + value, err := formatter. + WithPayload(value). + WithTemplate(template). + WithData("FieldName", name). + Render() + if err != nil { + return err + } + + namedArgs[name] = value + } + + _, err = stmt.QueryContext(ctx, namedArgs) + return err } diff --git a/pkg/storage/postgres/postgres_test.go b/pkg/storage/postgres/postgres_test.go index 0f61427..bd4463d 100644 --- a/pkg/storage/postgres/postgres_test.go +++ b/pkg/storage/postgres/postgres_test.go @@ -1,19 +1,22 @@ package postgres import ( - "database/sql" + "context" "fmt" "os" "testing" + "atomys.codes/webhooked/pkg/formatting" + "github.com/jmoiron/sqlx" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) type PostgresSetupTestSuite struct { suite.Suite - client *sql.DB + client *sqlx.DB databaseUrl string + ctx context.Context } // Create Table for running test @@ -29,12 +32,18 @@ func (suite *PostgresSetupTestSuite) BeforeTest(suiteName, testName string) { os.Getenv("POSTGRES_DB"), ) - if suite.client, err = sql.Open("postgres", suite.databaseUrl); err != nil { + if suite.client, err = sqlx.Open("postgres", suite.databaseUrl); err != nil { suite.T().Error(err) } if _, err := suite.client.Query("CREATE TABLE test (test_field TEXT)"); err != nil { suite.T().Error(err) } + + suite.ctx = formatting.ToContext( + context.Background(), + formatting.New().WithTemplate("{{.}}"), + ) + } // Delete Table after test @@ -61,6 +70,27 @@ func (suite *PostgresSetupTestSuite) TestPostgresNewStorage() { "dataField": "test_field", }) assert.NoError(suite.T(), err) + + _, err = NewStorage(map[string]interface{}{ + "databaseUrl": suite.databaseUrl, + "tableName": "test", + "useFormattingToPerformQuery": true, + }) + assert.Error(suite.T(), err) + + _, err = NewStorage(map[string]interface{}{ + "databaseUrl": suite.databaseUrl, + "useFormattingToPerformQuery": true, + "query": "", + }) + assert.Error(suite.T(), err) + + _, err = NewStorage(map[string]interface{}{ + "databaseUrl": suite.databaseUrl, + "useFormattingToPerformQuery": true, + "query": "INSERT INTO test (test_field) VALUES ('$field')", + }) + assert.NoError(suite.T(), err) } func (suite *PostgresSetupTestSuite) TestPostgresPush() { @@ -69,7 +99,7 @@ func (suite *PostgresSetupTestSuite) TestPostgresPush() { "tableName": "Not Exist", "dataField": "Not exist", }) - err := newClient.Push("Hello") + err := newClient.Push(suite.ctx, []byte("Hello")) assert.Error(suite.T(), err) newClient, err = NewStorage(map[string]interface{}{ @@ -79,10 +109,39 @@ func (suite *PostgresSetupTestSuite) TestPostgresPush() { }) assert.NoError(suite.T(), err) - err = newClient.Push("Hello") + err = newClient.Push(suite.ctx, []byte("Hello")) assert.NoError(suite.T(), err) } +func (suite *PostgresSetupTestSuite) TestPostgresPushNewFormattedQuery() { + newClient, err := NewStorage(map[string]interface{}{ + "databaseUrl": suite.databaseUrl, + "useFormattingToPerformQuery": true, + "query": "INSERT INTO test (test_field) VALUES (:field)", + "args": map[string]string{ + "field": "{{.Payload}}", + }, + }) + assert.NoError(suite.T(), err) + + fakePayload := []byte("A strange payload") + err = newClient.Push( + suite.ctx, + fakePayload, + ) + assert.NoError(suite.T(), err) + + rows, err := suite.client.Query("SELECT test_field FROM test") + assert.NoError(suite.T(), err) + + var result string + for rows.Next() { + err := rows.Scan(&result) + assert.NoError(suite.T(), err) + } + assert.Equal(suite.T(), string(fakePayload), result) +} + func TestRunPostgresPush(t *testing.T) { if testing.Short() { t.Skip("postgresql testing is skiped in short version of test") diff --git a/pkg/storage/rabbitmq/rabbitmq.go b/pkg/storage/rabbitmq/rabbitmq.go index a51610d..0d63b99 100644 --- a/pkg/storage/rabbitmq/rabbitmq.go +++ b/pkg/storage/rabbitmq/rabbitmq.go @@ -1,8 +1,8 @@ package rabbitmq import ( + "context" "errors" - "fmt" "time" "github.com/rs/zerolog/log" @@ -105,7 +105,7 @@ func (c *storage) Name() string { // A run is made from external caller // @param value that will be pushed // @return an error if the push failed -func (c *storage) Push(value interface{}) error { +func (c *storage) Push(ctx context.Context, value []byte) error { for attempt := 0; attempt < maxAttempt; attempt++ { err := c.channel.Publish( c.config.Exchange, @@ -114,7 +114,7 @@ func (c *storage) Push(value interface{}) error { c.config.Immediate, amqp.Publishing{ ContentType: c.config.ContentType(), - Body: []byte(fmt.Sprintf("%v", value)), + Body: value, }) if err != nil { diff --git a/pkg/storage/rabbitmq/rabbitmq_test.go b/pkg/storage/rabbitmq/rabbitmq_test.go index 849b54f..3538956 100644 --- a/pkg/storage/rabbitmq/rabbitmq_test.go +++ b/pkg/storage/rabbitmq/rabbitmq_test.go @@ -1,6 +1,7 @@ package rabbitmq import ( + "context" "fmt" "os" "testing" @@ -68,7 +69,7 @@ func (suite *RabbitMQSetupTestSuite) TestRabbitMQPush() { }) assert.NoError(suite.T(), err) - err = newClient.Push("Hello") + err = newClient.Push(context.Background(), []byte("Hello")) assert.NoError(suite.T(), err) } @@ -106,9 +107,9 @@ func (suite *RabbitMQSetupTestSuite) TestReconnect() { }) assert.NoError(suite.T(), err) - assert.NoError(suite.T(), newClient.Push("Hello")) + assert.NoError(suite.T(), newClient.Push(context.Background(), []byte("Hello"))) assert.NoError(suite.T(), newClient.client.Close()) - assert.NoError(suite.T(), newClient.Push("Hello")) + assert.NoError(suite.T(), newClient.Push(context.Background(), []byte("Hello"))) assert.NoError(suite.T(), newClient.channel.Close()) - assert.NoError(suite.T(), newClient.Push("Hello")) + assert.NoError(suite.T(), newClient.Push(context.Background(), []byte("Hello"))) } diff --git a/pkg/storage/redis/redis.go b/pkg/storage/redis/redis.go index b71ef74..836caaa 100644 --- a/pkg/storage/redis/redis.go +++ b/pkg/storage/redis/redis.go @@ -12,7 +12,6 @@ import ( type storage struct { client *redis.Client config *config - ctx context.Context } type config struct { @@ -33,7 +32,6 @@ func NewStorage(configRaw map[string]interface{}) (*storage, error) { newClient := storage{ config: &config{}, - ctx: context.Background(), } if err := valuable.Decode(configRaw, &newClient.config); err != nil { @@ -50,7 +48,7 @@ func NewStorage(configRaw map[string]interface{}) (*storage, error) { ) // Ping Redis for testing config - if err := newClient.client.Ping(newClient.ctx).Err(); err != nil { + if err := newClient.client.Ping(context.Background()).Err(); err != nil { return nil, err } @@ -67,8 +65,8 @@ func (c storage) Name() string { // A run is made from external caller // @param value that will be pushed // @return an error if the push failed -func (c storage) Push(value interface{}) error { - if err := c.client.RPush(c.ctx, c.config.Key, value).Err(); err != nil { +func (c storage) Push(ctx context.Context, value []byte) error { + if err := c.client.RPush(ctx, c.config.Key, value).Err(); err != nil { return err } diff --git a/pkg/storage/redis/redis_test.go b/pkg/storage/redis/redis_test.go index 76498ca..0420049 100644 --- a/pkg/storage/redis/redis_test.go +++ b/pkg/storage/redis/redis_test.go @@ -1,6 +1,7 @@ package redis import ( + "context" "os" "testing" @@ -44,10 +45,7 @@ func (suite *RedisSetupTestSuite) TestRedisPush() { }) assert.NoError(suite.T(), err) - err = newClient.Push(func() {}) - assert.Error(suite.T(), err) - - err = newClient.Push("Hello") + err = newClient.Push(context.Background(), []byte("Hello")) assert.NoError(suite.T(), err) } diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index a3e7c97..d446a4e 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -1,6 +1,7 @@ package storage import ( + "context" "fmt" "atomys.codes/webhooked/pkg/storage/postgres" @@ -16,7 +17,7 @@ type Pusher interface { // Will be unique across all storages Name() string // Method call when insert new data in the storage - Push(value interface{}) error + Push(ctx context.Context, value []byte) error } // Load will fetch and return the built-in storage based on the given diff --git a/tests/webhooks.tests.yml b/tests/webhooks.tests.yml index 5aa523c..41f6481 100644 --- a/tests/webhooks.tests.yml +++ b/tests/webhooks.tests.yml @@ -28,4 +28,20 @@ specs: }, "payload": {{ .Payload }} } - storage: [] \ No newline at end of file + storage: + - type: postgres + specs: + databaseUrl: 'postgresql://postgres:postgres@postgres:5432/postgres' + useFormattingToPerformQuery: true + query: | + INSERT INTO webhooks (payload, config, storage, metadata) VALUES (:payload, :config, :storage, :metadata) + args: + payload: '{{ .Payload }}' + config: '{{ toJson .Config }}' + storage: '{{ toJson .Storage }}' + metadata: | + { + "model": "{{ .Request.Header | getHeader "X-Model" }}", + "event": "{{ .Request.Header | getHeader "X-Event" }}", + "deliveryID": "{{ .Request.Header | getHeader "X-Delivery" | default "unknown" }}" + } \ No newline at end of file