feat: Formating Module

.github/workflows/tests.yaml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        goVersion: [ '1.16', '1.17' ]
+        goVersion: [ '1.16', '1.17', '1.18' ]
     steps:
     - name: Checkout project
       uses: actions/checkout@v3
@@ -66,6 +66,7 @@ jobs:
             echo "Failed"
             exit 1
         fi
+    - uses: codecov/codecov-action@v2
     - name: Run Go Build
       run: |
         go build -o /tmp/applications-test-units

README.md

@@ -2,6 +2,8 @@
 
 <p align="center"><a href="https://github.com/42Atomys/webhooked/actions/workflows/release.yaml"><img src="https://github.com/42Atomys/webhooked/actions/workflows/release.yaml/badge.svg" alt="Release 🎉"></a>
 <a href="https://goreportcard.com/report/atomys.codes/webhooked"><img src="https://goreportcard.com/badge/atomys.codes/webhooked" /></a>
+<a href="https://codeclimate.com/github/42Atomys/webhooked"><img alt="Code Climate maintainability" src="https://img.shields.io/codeclimate/maintainability/42Atomys/webhooked"></a>
+<a href="https://codecov.io/gh/42Atomys/webhooked"><img alt="Codecov" src="https://img.shields.io/codecov/c/gh/42Atomys/webhooked?token=NSUZMDT9M9"></a>
 <img src="https://img.shields.io/github/v/release/42atomys/webhooked?label=last%20release" alt="GitHub release (latest by date)">
 <img src="https://img.shields.io/github/contributors/42Atomys/webhooked?color=blueviolet" alt="GitHub contributors">
 <img src="https://img.shields.io/github/stars/42atomys/webhooked?color=blueviolet" alt="GitHub Repo stars">
@@ -21,7 +23,7 @@ This is exactly what `Webhooked` does !
 
 ## Roadmap
 
-I am actively working on this project to release a stable version by the **end of March 2022**
+I am actively working on this project in order to release a stable version for **mid-2022**
 
 ![Roadmap](/.github/profile/roadmap.png)
 
@@ -60,6 +62,28 @@ specs:
         values: ['foo', 'bar']
         valueFrom:
           envRef: SECRET_TOKEN
+
+  # Formatting allows you to apply a custom format to the payload received
+  # before send it to the storage. You can use built-in helper function to
+  # format it as you want. (Optional)
+  #
+  # Per default the format applied is: "{{ .Payload }}"
+  # 
+  # THIS IS AN ADVANCED FEATURE :
+  # Be careful when using this feature, the slightest error in format can
+  # result in DEFFINITIVE loss of the collected data. Make sure your template is
+  # correct before applying it in production.
+  formatting:
+    templateString: |
+      {
+        "config": "{{ toJson .Config }}",
+        "metadata": {
+          "specName": "{{ .Spec.Name }}",
+          "deliveryID": "{{ .Request.Header | getHeader "X-Delivery" | default "unknown" }}"
+        },
+        "payload": {{ .Payload }}
+      }
+
   # Storage allows you to list where you want to store the raw payloads
   # received by webhooked. You can add an unlimited number of storages, webhooked
   # will store in **ALL** the listed storages
@@ -68,6 +92,9 @@ specs:
   # on the `example-webhook` Redis Key on the Database 0
   storage:
   - type: redis
+    # You can apply a specific formatting per storage (Optional)
+    formatting: {}
+    # Storage specification
     specs:
       host: redis.default.svc.cluster.local
       port: 6379
@@ -77,7 +104,9 @@ specs:
 
 More informations about security pipeline available on wiki : [Configuration/Security](https://github.com/42Atomys/webhooked/wiki/Security)
 
-More informations about storages available on wiki : [Configuration/Storages](https://github.com/42Atomys/webhooked/wiki/Configuration-Storages)
+More informations about storages available on wiki : [Configuration/Storages](https://github.com/42Atomys/webhooked/wiki/Storages)
+
+More informations about formatting available on wiki : [Configuration/Formatting](https://github.com/42Atomys/webhooked/wiki/Formatting)
 
 ### Step 2 : Launch it 🚀
 ### With Kubernetes

config/webhooks.example.yml

@@ -12,7 +12,7 @@ specs:
   - compare:
       inputs:
       - name: first
-        value: '{{ Outputs.header.value }}'
+        value: '{{ .Outputs.header.value }}'
       - name: second
         valueFrom:
           envRef: SECRET_TOKEN

examples/kubernetes/deployment.yaml

@@ -22,7 +22,7 @@ data:
       - compare:
           inputs:
           - name: first
-            value: '{{ Outputs.header.value }}'
+            value: '{{ .Outputs.header.value }}'
           - name: second
             valueFrom:
               envRef: SECRET_TOKEN
@@ -40,7 +40,7 @@ metadata:
   name: webhooked
   labels:
     app.kubernetes.io/name: webhooked
-    app.kubernetes.io/version: '0.4'
+    app.kubernetes.io/version: '0.6'
 spec:
   selector:
     matchLabels:
@@ -52,7 +52,7 @@ spec:
     spec:
       containers:
       - name: webhooked
-        image: atomys/webhooked:0.4
+        image: atomys/webhooked:0.6
         imagePullPolicy: IfNotPresent
         env:
           - name: SECRET_TOKEN

go.mod

@@ -1,6 +1,6 @@
 module atomys.codes/webhooked
 
-go 1.17
+go 1.18
 
 require (
 	github.com/go-redis/redis/v8 v8.11.5

internal/config/configuration.go

@@ -1,8 +1,11 @@
 package config
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
+	"io"
+	"os"
 
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/viper"
@@ -15,6 +18,9 @@ var (
 	currentConfig = &Configuration{}
 	// ErrSpecNotFound is returned when the spec is not found
 	ErrSpecNotFound = errors.New("spec not found")
+	// defaultTemplate is the default template for the payload
+	// when no template is defined
+	defaultTemplate = `{{ .Payload }}`
 )
 
 // Load loads the configuration from the viper configuration file
@@ -30,6 +36,10 @@ func Load() error {
 			return err
 		}
 
+		if spec.Formatting, err = loadTemplate(spec.Formatting, nil); err != nil {
+			return fmt.Errorf("configured storage for %s received an error: %s", spec.Name, err.Error())
+		}
+
 		if err = loadStorage(spec); err != nil {
 			return fmt.Errorf("configured storage for %s received an error: %s", spec.Name, err.Error())
 		}
@@ -95,12 +105,62 @@ func loadStorage(spec *WebhookSpec) (err error) {
 		if err != nil {
 			return fmt.Errorf("storage %s cannot be loaded properly: %s", s.Type, err.Error())
 		}
+
+		if s.Formatting, err = loadTemplate(s.Formatting, spec.Formatting); err != nil {
+			return fmt.Errorf("storage %s cannot be loaded properly: %s", s.Type, err.Error())
+		}
 	}
 
 	log.Debug().Msgf("%d storages loaded for spec %s", len(spec.Storage), spec.Name)
 	return
 }
 
+// loadTemplate loads the template for the given `spec`. When no spec is defined
+// we try to load the template from the parentSpec and fallback to the default
+// template if parentSpec is not given.
+func loadTemplate(spec, parentSpec *FormattingSpec) (*FormattingSpec, error) {
+	if spec == nil {
+		spec = &FormattingSpec{}
+	}
+
+	if spec.TemplateString != "" {
+		spec.Template = spec.TemplateString
+		return spec, nil
+	}
+
+	if spec.TemplatePath != "" {
+		file, err := os.OpenFile(spec.TemplatePath, os.O_RDONLY, 0666)
+		if err != nil {
+			return spec, err
+		}
+		defer file.Close()
+
+		var buffer bytes.Buffer
+		_, err = io.Copy(&buffer, file)
+		if err != nil {
+			return spec, err
+		}
+
+		spec.Template = buffer.String()
+		return spec, nil
+	}
+
+	if parentSpec != nil {
+		if parentSpec.Template == "" {
+			var err error
+			parentSpec, err = loadTemplate(parentSpec, nil)
+			if err != nil {
+				return spec, err
+			}
+		}
+		spec.Template = parentSpec.Template
+	} else {
+		spec.Template = defaultTemplate
+	}
+
+	return spec, nil
+}
+
 // Current returns the aftual configuration
 func Current() *Configuration {
 	return currentConfig

internal/config/configuration_test.go

@@ -170,11 +170,11 @@ func TestLoadSecurityFactory(t *testing.T) {
 	for _, test := range tests {
 		err := loadSecurityFactory(test.input)
 		if test.wantErr {
-			assert.Error(err)
+			assert.Error(err, test.name)
 		} else {
-			assert.NoError(err)
+			assert.NoError(err, test.name)
 		}
-		assert.Equal(test.input.SecurityPipeline.FactoryCount(), test.wantLen)
+		assert.Equal(test.input.SecurityPipeline.FactoryCount(), test.wantLen, test.name)
 	}
 }
 
@@ -183,15 +183,13 @@ func TestLoadStorage(t *testing.T) {
 
 	tests := []struct {
 		name        string
-		storageName string
 		input       *WebhookSpec
 		wantErr     bool
 		wantStorage bool
 	}{
-		{"no spec", "", &WebhookSpec{Name: "test"}, false, false},
+		{"no spec", &WebhookSpec{Name: "test"}, false, false},
 		{
 			"full valid storage",
-			"connection invalid must return an error",
 			&WebhookSpec{
 				Name: "test",
 				Storage: []*StorageSpec{
@@ -201,6 +199,7 @@ func TestLoadStorage(t *testing.T) {
 							"host": "localhost",
 							"port": 0,
 						},
+						Formatting: &FormattingSpec{TemplateString: "null"},
 					},
 				},
 			},
@@ -209,7 +208,6 @@ func TestLoadStorage(t *testing.T) {
 		},
 		{
 			"empty storage configuration",
-			"",
 			&WebhookSpec{
 				Name:    "test",
 				Storage: []*StorageSpec{},
@@ -219,7 +217,6 @@ func TestLoadStorage(t *testing.T) {
 		},
 		{
 			"invalid storage name in configuration",
-			"",
 			&WebhookSpec{
 				Name: "test",
 				Storage: []*StorageSpec{
@@ -234,14 +231,92 @@ func TestLoadStorage(t *testing.T) {
 	for _, test := range tests {
 		err := loadStorage(test.input)
 		if test.wantErr {
-			assert.Error(err)
+			assert.Error(err, test.name)
 		} else {
-			assert.NoError(err)
+			assert.NoError(err, test.name)
 		}
 
 		if test.wantStorage && assert.Len(test.input.Storage, 1, "no storage is loaded for test %s", test.name) {
 			s := test.input.Storage[0]
-			assert.NotNil(s)
+			assert.NotNil(s, test.name)
+		}
+	}
+}
+
+func Test_loadTemplate(t *testing.T) {
+	tests := []struct {
+		name         string
+		input        *FormattingSpec
+		parentSpec   *FormattingSpec
+		wantErr      bool
+		wantTemplate string
+	}{
+		{
+			"no template",
+			nil,
+			nil,
+			false,
+			defaultTemplate,
+		},
+		{
+			"template string",
+			&FormattingSpec{TemplateString: "{{ .Request.Method }}"},
+			nil,
+			false,
+			"{{ .Request.Method }}",
+		},
+		{
+			"template file",
+			&FormattingSpec{TemplatePath: "../../tests/simple_template.tpl"},
+			nil,
+			false,
+			"{{ .Request.Method }}",
+		},
+		{
+			"template file with template string",
+			&FormattingSpec{TemplatePath: "../../tests/simple_template.tpl", TemplateString: "{{ .Request.Path }}"},
+			nil,
+			false,
+			"{{ .Request.Path }}",
+		},
+		{
+			"no template with not loaded parent",
+			nil,
+			&FormattingSpec{TemplateString: "{{ .Request.Method }}"},
+			false,
+			"{{ .Request.Method }}",
+		},
+		{
+			"no template with loaded parent",
+			nil,
+			&FormattingSpec{Template: "{{ .Request.Method }}", TemplateString: "{{ .Request.Path }}"},
+			false,
+			"{{ .Request.Method }}",
+		},
+		{
+			"no template with unloaded parent and error",
+			nil,
+			&FormattingSpec{TemplatePath: "//invalid//path//"},
+			true,
+			"",
+		},
+		{
+			"template file not found",
+			&FormattingSpec{TemplatePath: "//invalid//path//"},
+			nil,
+			true,
+			"",
+		},
+	}
+
+	for _, test := range tests {
+		tmpl, err := loadTemplate(test.input, test.parentSpec)
+		if test.wantErr {
+			assert.Error(t, err, test.name)
+		} else {
+			assert.NoError(t, err, test.name)
 		}
+		assert.NotNil(t, tmpl, test.name)
+		assert.Equal(t, test.wantTemplate, tmpl.Template, test.name)
 	}
 }

internal/config/specification.go

@@ -4,3 +4,13 @@ package config
 func (s WebhookSpec) HasSecurity() bool {
 	return s.SecurityPipeline != nil && s.SecurityPipeline.HasFactories()
 }
+
+// HasGlobalFormatting returns true if the spec has a global formatting
+func (s WebhookSpec) HasGlobalFormatting() bool {
+	return s.Formatting != nil && (s.Formatting.TemplatePath != "" || s.Formatting.TemplateString != "")
+}
+
+// HasFormatting returns true if the storage spec has a formatting
+func (s StorageSpec) HasFormatting() bool {
+	return s.Formatting != nil && (s.Formatting.TemplatePath != "" || s.Formatting.TemplateString != "")
+}

internal/config/specification_test.go

@@ -12,3 +12,23 @@ func TestWebhookSpec_HasSecurity(t *testing.T) {
 	// assert.True(t, WebhookSpec{SecurityFactories: make([]*factory.Factory, 1)}.HasSecurity())
 	// assert.True(t, WebhookSpec{SecurityFactories: make([]*factory.Factory, 2)}.HasSecurity())
 }
+
+func TestWebhookSpec_HasGlobalFormatting(t *testing.T) {
+	assert.False(t, WebhookSpec{Formatting: nil}.HasGlobalFormatting())
+	assert.False(t, WebhookSpec{Formatting: &FormattingSpec{}}.HasGlobalFormatting())
+	assert.False(t, WebhookSpec{Formatting: &FormattingSpec{TemplatePath: ""}}.HasGlobalFormatting())
+	assert.False(t, WebhookSpec{Formatting: &FormattingSpec{TemplateString: ""}}.HasGlobalFormatting())
+	assert.False(t, WebhookSpec{Formatting: &FormattingSpec{TemplatePath: "", TemplateString: ""}}.HasGlobalFormatting())
+	assert.True(t, WebhookSpec{Formatting: &FormattingSpec{TemplatePath: "/_tmp/invalid_path", TemplateString: ""}}.HasGlobalFormatting())
+	assert.True(t, WebhookSpec{Formatting: &FormattingSpec{TemplatePath: "/_tmp/invalid_path", TemplateString: "{{}}"}}.HasGlobalFormatting())
+}
+
+func TestWebhookSpec_HasFormatting(t *testing.T) {
+	assert.False(t, StorageSpec{Formatting: nil}.HasFormatting())
+	assert.False(t, StorageSpec{Formatting: &FormattingSpec{}}.HasFormatting())
+	assert.False(t, StorageSpec{Formatting: &FormattingSpec{TemplatePath: ""}}.HasFormatting())
+	assert.False(t, StorageSpec{Formatting: &FormattingSpec{TemplateString: ""}}.HasFormatting())
+	assert.False(t, StorageSpec{Formatting: &FormattingSpec{TemplatePath: "", TemplateString: ""}}.HasFormatting())
+	assert.True(t, StorageSpec{Formatting: &FormattingSpec{TemplatePath: "/_tmp/invalid_path", TemplateString: ""}}.HasFormatting())
+	assert.True(t, StorageSpec{Formatting: &FormattingSpec{TemplatePath: "/_tmp/invalid_path", TemplateString: "{{}}"}}.HasFormatting())
+}

internal/config/structs.go

@@ -9,19 +9,19 @@ import (
 // defined in the webhooks yaml file
 type Configuration struct {
 	// APIVerion is the version of the API that will be used
-	APIVersion string `mapstructure:"apiVersion"`
+	APIVersion string `mapstructure:"apiVersion" json:"apiVersion"`
 	// Observability is the configuration for observability
-	Observability Observability `mapstructure:"observability"`
+	Observability Observability `mapstructure:"observability" json:"observability"`
 	// Specs is the configuration for the webhooks specs
-	Specs []*WebhookSpec `mapstructure:"specs"`
+	Specs []*WebhookSpec `mapstructure:"specs" json:"specs"`
 }
 
 // Observability is the struct contains the configuration for observability
 // defined in the webhooks yaml file.
 type Observability struct {
 	// MetricsEnabled is the flag to enable or disable the prometheus metrics
 	// endpoint and expose the metrics
-	MetricsEnabled bool `mapstructure:"metricsEnabled"`
+	MetricsEnabled bool `mapstructure:"metricsEnabled" json:"metricsEnabled"`
 }
 
 // WebhookSpec is the struct contains the configuration for a webhook spec
@@ -30,22 +30,28 @@ type WebhookSpec struct {
 	// Name is the name of the webhook spec. It must be unique in the configuration
 	// file. It is used to identify the webhook spec in the configuration file
 	// and is defined by the user
-	Name string `mapstructure:"name"`
+	Name string `mapstructure:"name" json:"name"`
 	// EntrypointURL is the URL of the entrypoint of the webhook spec. It must
 	// be unique in the configuration file. It is defined by the user
 	// It is used to identify the webhook spec when receiving a request
-	EntrypointURL string `mapstructure:"entrypointUrl"`
+	EntrypointURL string `mapstructure:"entrypointUrl" json:"entrypointUrl"`
 	// Security is the configuration for the security of the webhook spec
 	// It is defined by the user and can be empty. See HasSecurity() method
 	// to know if the webhook spec has security
-	Security []map[string]Security `mapstructure:"security"`
+	Security []map[string]Security `mapstructure:"security" json:"-"`
+	// Format is used to define the payload format sent by the webhook spec
+	// to all storages. Each storage can have its own format. When this
+	// configuration is empty, the default formatting setting is used (body as JSON)
+	// It is defined by the user and can be empty. See HasGlobalFormatting() method
+	// to know if the webhook spec has format
+	Formatting *FormattingSpec `mapstructure:"formatting" json:"-"`
 	// SecurityPipeline is the security pipeline of the webhook spec
 	// It is defined by the configuration loader. This field is not defined
 	// by the user and cannot be overridden
-	SecurityPipeline *factory.Pipeline `mapstructure:"-"`
+	SecurityPipeline *factory.Pipeline `mapstructure:"-" json:"-"`
 	// Storage is the configuration for the storage of the webhook spec
 	// It is defined by the user and can be empty.
-	Storage []*StorageSpec `mapstructure:"storage"`
+	Storage []*StorageSpec `mapstructure:"storage" json:"-"`
 }
 
 // Security is the struct contains the configuration for a security
@@ -70,11 +76,32 @@ type Security struct {
 type StorageSpec struct {
 	// Type is the type of the storage. It must be a valid storage type
 	// defined in the storage package.
-	Type string `mapstructure:"type"`
+	Type string `mapstructure:"type" json:"type"`
 	// Specs is the configuration for the storage. It is defined by the user
 	// following the storage type specification
-	Specs map[string]interface{} `mapstructure:"specs"`
+	// NOTE: this field is hidden for json to prevent mistake of the user
+	//       when he use the custom formatting option and leak credentials
+	Specs map[string]interface{} `mapstructure:"specs" json:"-"`
+	// Format is used to define the payload format sent by the webhook spec
+	// to this storage. If not defined, the format of the webhook spec is
+	// used.
+	// It is defined by the user and can be empty. See HasFormatting() method
+	// to know if the webhook spec has format
+	Formatting *FormattingSpec `mapstructure:"formatting" json:"-"`
 	// Client is the storage client. It is defined by the configuration loader
 	// and cannot be overridden
-	Client storage.Pusher `mapstructure:"-"`
+	Client storage.Pusher `mapstructure:"-" json:"-"`
+}
+
+// FormattingSpec is the struct contains the configuration to formatting the
+// payload of the webhook spec. The field TempalteString is prioritized
+// over the field TemplatePath when both are defined.
+type FormattingSpec struct {
+	// TemplatePath is the path to the template used to formatting the payload
+	TemplatePath string `mapstructure:"templatePath"`
+	// TemplateString is a plaintext template used to formatting the payload
+	TemplateString string `mapstructure:"templateString"`
+	// ResolvedTemplate is the template after resolving the template variables
+	// It is defined by the configuration loader and cannot be overridden
+	Template string `mapstructure:"-"`
 }

internal/server/v1alpha1/handlers.go

@@ -11,6 +11,7 @@ import (
 	"github.com/rs/zerolog/log"
 
 	"atomys.codes/webhooked/internal/config"
+	"atomys.codes/webhooked/pkg/formatting"
 )
 
 // Server is the server instance for the v1alpha1 version
@@ -101,9 +102,21 @@ func webhookService(s *Server, spec *config.WebhookSpec, r *http.Request) (err e
 		}
 	}
 
-	log.Debug().Msgf("store following data: %+v", string(data))
 	for _, storage := range spec.Storage {
-		if err := storage.Client.Push(string(data)); err != nil {
+		str, err := formatting.
+			NewTemplateData(storage.Formatting.Template).
+			WithRequest(r).
+			WithPayload(data).
+			WithSpec(spec).
+			WithStorage(storage).
+			WithConfig().
+			Render()
+		if err != nil {
+			return err
+		}
+
+		log.Debug().Msgf("store following data: %+v", str)
+		if err := storage.Client.Push(str); err != nil {
 			return err
 		}
 		log.Debug().Str("storage", storage.Client.Name()).Msgf("stored successfully")

internal/valuable/valuable.go

@@ -40,7 +40,7 @@ type ValueFromSource struct {
 func (v *Valuable) Validate() error {
 	if v.ValueFrom != nil && v.ValueFrom.EnvRef != nil {
 		if _, ok := os.LookupEnv(*v.ValueFrom.EnvRef); !ok {
-			return fmt.Errorf("enviroment variable %s not found", *v.ValueFrom.EnvRef)
+			return fmt.Errorf("environment variable %s not found", *v.ValueFrom.EnvRef)
 		}
 	}
 

pkg/formatting/format.go

@@ -0,0 +1,90 @@
+package formatting
+
+import (
+	"bytes"
+	"fmt"
+	"net/http"
+	"text/template"
+
+	"github.com/rs/zerolog/log"
+
+	"atomys.codes/webhooked/internal/config"
+)
+
+type TemplateData struct {
+	tmplString string
+	data       map[string]interface{}
+}
+
+// NewTemplateData returns a new TemplateData instance. It takes the template
+// string as a parameter. The template string is the string that will be used
+// to render the template. The data is the map of data that will be used to
+// render the template.
+func NewTemplateData(tmplString string) *TemplateData {
+	return &TemplateData{
+		tmplString: tmplString,
+		data:       make(map[string]interface{}),
+	}
+}
+
+// WithData adds a key-value pair to the data map. The key is the name of the
+// variable and the value is the value of the variable.
+func (d *TemplateData) WithData(name string, data interface{}) *TemplateData {
+	d.data[name] = data
+	return d
+}
+
+// WithRequest adds a http.Request object to the data map. The key of request is
+// "Request".
+func (d *TemplateData) WithRequest(r *http.Request) *TemplateData {
+	d.WithData("Request", r)
+	return d
+}
+
+// WithPayload adds a payload to the data map. The key of payload is "Payload".
+// The payload is basically the body of the request.
+func (d *TemplateData) WithPayload(payload []byte) *TemplateData {
+	d.WithData("Payload", string(payload))
+	return d
+}
+
+// WithSpec adds a webhookspec to the data map. The key of spec is "Spec".
+func (d *TemplateData) WithSpec(spec *config.WebhookSpec) *TemplateData {
+	d.WithData("Spec", spec)
+	return d
+}
+
+// WithStorage adds a storage spec to the data map.
+// The key of storage is "Storage".
+func (d *TemplateData) WithStorage(spec *config.StorageSpec) *TemplateData {
+	d.WithData("Storage", spec)
+	return d
+}
+
+// WithConfig adds the current config to the data map.
+// The key of config is "Config".
+func (d *TemplateData) WithConfig() *TemplateData {
+	d.WithData("Config", config.Current())
+	return d
+}
+
+// Render returns the rendered template string. It takes the template string
+// from the TemplateData instance and the data stored in the TemplateData
+// instance. It returns an error if the template string is invalid or when
+// rendering the template fails.
+func (d *TemplateData) Render() (string, error) {
+	log.Debug().Msgf("rendering template: %s", d.tmplString)
+
+	t := template.New("formattingTmpl").Funcs(funcMap())
+	t, err := t.Parse(d.tmplString)
+	if err != nil {
+		return "", fmt.Errorf("error in your template: %s", err.Error())
+	}
+
+	buf := new(bytes.Buffer)
+	if err := t.Execute(buf, d.data); err != nil {
+		return "", fmt.Errorf("error while filling your template: %s", err.Error())
+	}
+
+	return buf.String(), nil
+}

pkg/formatting/format_test.go

@@ -0,0 +1,176 @@
+package formatting
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"atomys.codes/webhooked/internal/config"
+)
+
+func TestNewTemplateData(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpl := NewTemplateData("")
+	assert.NotNil(tmpl)
+	assert.Equal("", tmpl.tmplString)
+	assert.Equal(0, len(tmpl.data))
+
+	tmpl = NewTemplateData("{{ .Payload }}")
+	assert.NotNil(tmpl)
+	assert.Equal("{{ .Payload }}", tmpl.tmplString)
+	assert.Equal(0, len(tmpl.data))
+}
+
+func Test_WithData(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpl := NewTemplateData("").WithData("test", true)
+	assert.NotNil(tmpl)
+	assert.Equal("", tmpl.tmplString)
+	assert.Equal(1, len(tmpl.data))
+	assert.Equal(true, tmpl.data["test"])
+}
+
+func Test_WithRequest(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpl := NewTemplateData("").WithRequest(httptest.NewRequest("GET", "/", nil))
+	assert.NotNil(tmpl)
+	assert.Equal("", tmpl.tmplString)
+	assert.Equal(1, len(tmpl.data))
+	assert.Nil(tmpl.data["request"])
+	assert.NotNil(tmpl.data["Request"])
+	assert.Equal("GET", tmpl.data["Request"].(*http.Request).Method)
+}
+
+func Test_WithPayload(t *testing.T) {
+	assert := assert.New(t)
+
+	data, err := json.Marshal(map[string]interface{}{"test": "test"})
+	assert.Nil(err)
+
+	tmpl := NewTemplateData("").WithPayload(data)
+	assert.NotNil(tmpl)
+	assert.Equal("", tmpl.tmplString)
+	assert.Equal(1, len(tmpl.data))
+	assert.JSONEq(`{"test":"test"}`, tmpl.data["Payload"].(string))
+}
+
+func Test_WithSpec(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpl := NewTemplateData("").WithSpec(&config.WebhookSpec{Name: "test", Formatting: &config.FormattingSpec{}})
+	assert.NotNil(tmpl)
+	assert.Equal("", tmpl.tmplString)
+	assert.Equal(1, len(tmpl.data))
+	assert.Equal("test", tmpl.data["Spec"].(*config.WebhookSpec).Name)
+}
+
+func Test_WithStorage(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpl := NewTemplateData("").WithStorage(&config.StorageSpec{Type: "testing"})
+	assert.NotNil(tmpl)
+	assert.Equal("", tmpl.tmplString)
+	assert.Equal(1, len(tmpl.data))
+	assert.Equal("testing", tmpl.data["Storage"].(*config.StorageSpec).Type)
+}
+
+func Test_WithConfig(t *testing.T) {
+	assert := assert.New(t)
+
+	tmpl := NewTemplateData("").WithConfig()
+	assert.NotNil(tmpl)
+	assert.Equal("", tmpl.tmplString)
+	assert.Equal(1, len(tmpl.data))
+	assert.NotNil(tmpl.data["Config"])
+}
+
+func Test_Render(t *testing.T) {
+	assert := assert.New(t)
+
+	// Test with basic template
+	tmpl := NewTemplateData("{{ .Payload }}").WithPayload([]byte(`{"test": "test"}`))
+	assert.NotNil(tmpl)
+	assert.Equal("{{ .Payload }}", tmpl.tmplString)
+	assert.Equal(1, len(tmpl.data))
+	assert.JSONEq(`{"test":"test"}`, tmpl.data["Payload"].(string))
+
+	str, err := tmpl.Render()
+	assert.Nil(err)
+	assert.JSONEq("{\"test\":\"test\"}", str)
+
+	// Test with template with multiple data sources
+	// and complex template
+	req := httptest.NewRequest("GET", "/", nil)
+	req.Header.Set("X-Test", "test")
+
+	tmpl = NewTemplateData(`
+	{
+		"config": {{ toJson .Config }},
+		"spec": {{ toJson .Spec }},
+		"storage": {{ toJson .Storage }},
+		"metadata": {
+			"testID": "{{ .Request.Header | getHeader "X-Test" }}",
+			"deliveryID": "{{ .Request.Header | getHeader "X-Delivery" | default "unknown" }}"
+		},
+		"payload": {{ .Payload }}
+	}
+	`).
+		WithPayload([]byte(`{"test": "test"}`)).
+		WithRequest(req).
+		WithSpec(&config.WebhookSpec{Name: "test", EntrypointURL: "/webhooks/test", Formatting: &config.FormattingSpec{}}).
+		WithStorage(&config.StorageSpec{Type: "testing", Specs: map[string]interface{}{}}).
+		WithConfig()
+	assert.NotNil(tmpl)
+
+	str, err = tmpl.Render()
+	assert.Nil(err)
+	assert.JSONEq(`{
+		"config": {
+			"apiVersion":"",
+			"observability":{
+				"metricsEnabled":false
+			},
+			"specs": null
+		},
+		"spec": {
+			"name":"test",
+			"entrypointUrl": "/webhooks/test"
+		},
+		"storage": {
+			"type": "testing"
+		},
+		"metadata": {
+			"testID": "test",
+			"deliveryID": "unknown"
+		},
+		"payload": {
+			"test": "test"
+		}
+	}`, str)
+
+	// Test with template with template error
+	tmpl = NewTemplateData("{{ .Payload }")
+	assert.NotNil(tmpl)
+	assert.Equal("{{ .Payload }", tmpl.tmplString)
+
+	str, err = tmpl.Render()
+	assert.Error(err)
+	assert.Contains(err.Error(), "error in your template: ")
+	assert.Equal("", str)
+
+	// Test with template with data error
+	tmpl = NewTemplateData("{{ .Request.Method }}").WithRequest(nil)
+	assert.NotNil(tmpl)
+	assert.Equal("{{ .Request.Method }}", tmpl.tmplString)
+
+	str, err = tmpl.Render()
+	assert.Error(err)
+	assert.Contains(err.Error(), "error while filling your template: ")
+	assert.Equal("", str)
+}

pkg/formatting/functions.go

@@ -0,0 +1,116 @@
+package formatting
+
+import (
+	"encoding/json"
+	"net/http"
+	"reflect"
+	"text/template"
+
+	"github.com/rs/zerolog/log"
+)
+
+// funcMap is the map of functions that can be used in templates.
+// The key is the name of the function and the value is the function itself.
+// This is required for the template.New() function to parse the function.
+func funcMap() template.FuncMap {
+	return template.FuncMap{
+		// Core functions
+		"default":      dft,
+		"empty":        empty,
+		"coalesce":     coalesce,
+		"toJson":       toJson,
+		"toPrettyJson": toPrettyJson,
+		"ternary":      ternary,
+
+		// Headers manipulation functions
+		"getHeader": getHeader,
+	}
+}
+
+// dft returns the default value if the given value is empty.
+// If the given value is not empty, it is returned as is.
+func dft(dft interface{}, given ...interface{}) interface{} {
+
+	if empty(given) || empty(given[0]) {
+		return dft
+	}
+	return given[0]
+}
+
+// empty returns true if the given value is empty.
+// It supports any type.
+func empty(given interface{}) bool {
+	g := reflect.ValueOf(given)
+	if !g.IsValid() {
+		return true
+	}
+
+	switch g.Kind() {
+	default:
+		return g.IsNil()
+	case reflect.Array, reflect.Slice, reflect.Map, reflect.String:
+		return g.Len() == 0
+	case reflect.Bool:
+		return !g.Bool()
+	case reflect.Complex64, reflect.Complex128:
+		return g.Complex() == 0
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		return g.Int() == 0
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
+		return g.Uint() == 0
+	case reflect.Float32, reflect.Float64:
+		return g.Float() == 0
+	case reflect.Struct:
+		return false
+	}
+}
+
+// coalesce returns the first value not empty in the given list.
+// If all values are empty, it returns nil.
+func coalesce(v ...interface{}) interface{} {
+	for _, val := range v {
+		if !empty(val) {
+			return val
+		}
+	}
+	return nil
+}
+
+// toJson returns the given value as a JSON string.
+// If the given value is nil, it returns an empty string.
+func toJson(v interface{}) string {
+	output, err := json.Marshal(v)
+	if err != nil {
+		log.Error().Err(err).Msg("Failed to marshal to JSON")
+	}
+	return string(output)
+}
+
+// toPrettyJson returns the given value as a pretty JSON string indented with
+// 2 spaces. If the given value is nil, it returns an empty string.
+func toPrettyJson(v interface{}) string {
+	output, err := json.MarshalIndent(v, "", "  ")
+	if err != nil {
+		log.Error().Err(err).Msg("Failed to marshal to JSON")
+	}
+	return string(output)
+}
+
+// ternary returns `isTrue` if `condition` is true, otherwise returns `isFalse`.
+func ternary(isTrue interface{}, isFalse interface{}, confition bool) interface{} {
+	if confition {
+		return isTrue
+	}
+
+	return isFalse
+}
+
+// getHeader returns the value of the given header. If the header is not found,
+// it returns an empty string.
+func getHeader(name string, headers *http.Header) string {
+	if headers == nil {
+		log.Error().Msg("headers are nil. Returning empty string")
+		return ""
+	}
+	return headers.Get(name)
+}

pkg/formatting/functions_test.go

@@ -0,0 +1,109 @@
+package formatting
+
+import (
+	"net/http/httptest"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_funcMap(t *testing.T) {
+	assert := assert.New(t)
+
+	funcMap := funcMap()
+	assert.Contains(funcMap, "default")
+	assert.NotContains(funcMap, "dft")
+	assert.Contains(funcMap, "empty")
+	assert.Contains(funcMap, "coalesce")
+	assert.Contains(funcMap, "toJson")
+	assert.Contains(funcMap, "toPrettyJson")
+	assert.Contains(funcMap, "ternary")
+	assert.Contains(funcMap, "getHeader")
+}
+
+func Test_dft(t *testing.T) {
+	assert := assert.New(t)
+
+	assert.Equal("test", dft("default", "test"))
+	assert.Equal("default", dft("default", nil))
+	assert.Equal("default", dft("default", ""))
+}
+
+func Test_empty(t *testing.T) {
+	assert := assert.New(t)
+
+	assert.True(empty(""))
+	assert.True(empty(nil))
+	assert.False(empty("test"))
+	assert.False(empty(true))
+	assert.True(empty(false))
+	assert.True(empty(0 + 0i))
+	assert.False(empty(2 + 4i))
+	assert.True(empty([]int{}))
+	assert.False(empty([]int{1}))
+	assert.True(empty(map[string]string{}))
+	assert.False(empty(map[string]string{"test": "test"}))
+	assert.True(empty(map[string]interface{}{}))
+	assert.False(empty(map[string]interface{}{"test": "test"}))
+	assert.True(empty(0))
+	assert.False(empty(-1))
+	assert.False(empty(1))
+	assert.True(empty(uint32(0)))
+	assert.False(empty(uint32(1)))
+	assert.True(empty(float64(0.0)))
+	assert.False(empty(float64(1.0)))
+	assert.False(empty(struct{}{}))
+	assert.False(empty(struct{ Test string }{Test: "test"}))
+
+	ptr := &struct{ Test string }{Test: "test"}
+	assert.False(empty(ptr))
+}
+
+func Test_coalesce(t *testing.T) {
+	assert := assert.New(t)
+
+	assert.Equal("test", coalesce("test", "default"))
+	assert.Equal("default", coalesce("", "default"))
+	assert.Equal("default", coalesce(nil, "default"))
+	assert.Equal(nil, coalesce(nil, nil))
+}
+
+func Test_toJson(t *testing.T) {
+	assert := assert.New(t)
+
+	assert.Equal("{\"test\":\"test\"}", toJson(map[string]string{"test": "test"}))
+	assert.Equal("{\"test\":\"test\"}", toJson(map[string]interface{}{"test": "test"}))
+	assert.Equal("null", toJson(nil))
+	assert.Equal("", toJson(map[string]interface{}{"test": func() {}}))
+}
+
+func Test_toPrettyJson(t *testing.T) {
+	assert := assert.New(t)
+
+	assert.Equal("{\n  \"test\": \"test\"\n}", toPrettyJson(map[string]string{"test": "test"}))
+	assert.Equal("{\n  \"test\": \"test\"\n}", toPrettyJson(map[string]interface{}{"test": "test"}))
+	assert.Equal("null", toPrettyJson(nil))
+	assert.Equal("", toPrettyJson(map[string]interface{}{"test": func() {}}))
+}
+
+func Test_ternary(t *testing.T) {
+	assert := assert.New(t)
+
+	header := httptest.NewRecorder().Header()
+
+	header.Set("X-Test", "test")
+	assert.Equal("test", getHeader("X-Test", &header))
+	assert.Equal("", getHeader("X-Undefined", &header))
+	assert.Equal("", getHeader("", &header))
+	assert.Equal("", getHeader("", nil))
+}
+
+func Test_getHeader(t *testing.T) {
+	assert := assert.New(t)
+
+	assert.Equal(true, ternary(true, false, true))
+	assert.Equal(false, ternary(true, false, false))
+	assert.Equal("true string", ternary("true string", "false string", true))
+	assert.Equal("false string", ternary("true string", "false string", false))
+	assert.Equal(nil, ternary(nil, nil, false))
+}

pkg/storage/postgres/postgres.go

@@ -4,7 +4,6 @@ import (
 	"database/sql"
 	"fmt"
 
-	// Import postgres driver
 	_ "github.com/lib/pq"
 
 	"atomys.codes/webhooked/internal/valuable"
@@ -20,9 +19,9 @@ type storage struct {
 // config is the struct contains config for connect client
 // Run is made from internal caller
 type config struct {
-	DatabaseURL valuable.Valuable
-	TableName   string
-	DataField   string
+	DatabaseURL valuable.Valuable `json:"databaseUrl"`
+	TableName   string            `json:"tableName"`
+	DataField   string            `json:"dataField"`
 }
 
 // NewStorage is the function for create new Postgres client storage

pkg/storage/postgres/postgres_test.go

@@ -38,12 +38,12 @@ func (suite *PostgresSetupTestSuite) TestPostgresName() {
 
 func (suite *PostgresSetupTestSuite) TestPostgresNewStorage() {
 	_, err := NewStorage(map[string]interface{}{
-		"databaseURL": []int{1},
+		"databaseUrl": []int{1},
 	})
 	assert.Error(suite.T(), err)
 
 	_, err = NewStorage(map[string]interface{}{
-		"databaseURL": "postgresql://webhook:test@127.0.0.1:5432/webhook_db?sslmode=disable",
+		"databaseUrl": "postgresql://webhook:test@127.0.0.1:5432/webhook_db?sslmode=disable",
 		"tableName":   "test",
 		"dataField":   "test_field",
 	})
@@ -52,15 +52,15 @@ func (suite *PostgresSetupTestSuite) TestPostgresNewStorage() {
 
 func (suite *PostgresSetupTestSuite) TestPostgresPush() {
 	newClient, _ := NewStorage(map[string]interface{}{
-		"databaseURL": "postgresql://webhook:test@127.0.0.1:5432/webhook_db?sslmode=disable",
+		"databaseUrl": "postgresql://webhook:test@127.0.0.1:5432/webhook_db?sslmode=disable",
 		"tableName":   "Not Exist",
 		"dataField":   "Not exist",
 	})
 	err := newClient.Push("Hello")
 	assert.Error(suite.T(), err)
 
 	newClient, err = NewStorage(map[string]interface{}{
-		"databaseURL": "postgresql://webhook:test@127.0.0.1:5432/webhook_db?sslmode=disable",
+		"databaseUrl": "postgresql://webhook:test@127.0.0.1:5432/webhook_db?sslmode=disable",
 		"tableName":   "test",
 		"dataField":   "test_field",
 	})

pkg/storage/rabbitmq/rabbitmq.go

@@ -4,8 +4,9 @@ import (
 	"bytes"
 	"encoding/gob"
 
-	"atomys.codes/webhooked/internal/valuable"
 	"github.com/streadway/amqp"
+
+	"atomys.codes/webhooked/internal/valuable"
 )
 
 // storage is the struct contains client and config
@@ -20,15 +21,15 @@ type storage struct {
 // config is the struct contains config for connect client
 // Run is made from internal caller
 type config struct {
-	DatabaseURL      valuable.Valuable
-	QueueName        string
-	Durable          bool
-	DeleteWhenUnused bool
-	Exclusive        bool
-	NoWait           bool
-	Mandatory        bool
-	Immediate        bool
-	Exchange         string
+	DatabaseURL      valuable.Valuable `json:"databaseUrl"`
+	QueueName        string            `json:"queueName"`
+	Durable          bool              `json:"durable"`
+	DeleteWhenUnused bool              `json:"deleteWhenUnused"`
+	Exclusive        bool              `json:"exclusive"`
+	NoWait           bool              `json:"noWait"`
+	Mandatory        bool              `json:"mandatory"`
+	Immediate        bool              `json:"immediate"`
+	Exchange         string            `json:"exchange"`
 }
 
 // NewStorage is the function for create new RabbitMQ client storage

pkg/storage/rabbitmq/rabbitmq_test.go

@@ -18,12 +18,12 @@ func (suite *RabbitMQSetupTestSuite) TestRabbitMQName() {
 
 func (suite *RabbitMQSetupTestSuite) TestRabbitMQNewStorage() {
 	_, err := NewStorage(map[string]interface{}{
-		"databaseURL": []int{1},
+		"databaseUrl": []int{1},
 	})
 	assert.Error(suite.T(), err)
 
 	_, err = NewStorage(map[string]interface{}{
-		"databaseURL":      "amqp://user:password@127.0.0.1:5672",
+		"databaseUrl":      "amqp://user:password@127.0.0.1:5672",
 		"queueName":        "hello",
 		"durable":          false,
 		"deleteWhenUnused": false,
@@ -37,7 +37,7 @@ func (suite *RabbitMQSetupTestSuite) TestRabbitMQNewStorage() {
 
 func (suite *RabbitMQSetupTestSuite) TestRabbitMQPush() {
 	newClient, _ := NewStorage(map[string]interface{}{
-		"databaseURL":      "amqp://user:password@127.0.0.1:5672",
+		"databaseUrl":      "amqp://user:password@127.0.0.1:5672",
 		"queueName":        "hello",
 		"durable":          false,
 		"deleteWhenUnused": false,
@@ -50,7 +50,7 @@ func (suite *RabbitMQSetupTestSuite) TestRabbitMQPush() {
 	assert.Error(suite.T(), err)
 
 	newClient, err = NewStorage(map[string]interface{}{
-		"databaseURL":      "amqp://user:password@127.0.0.1:5672",
+		"databaseUrl":      "amqp://user:password@127.0.0.1:5672",
 		"queueName":        "hello",
 		"durable":          false,
 		"deleteWhenUnused": false,

pkg/storage/redis/redis.go

@@ -16,12 +16,12 @@ type storage struct {
 }
 
 type config struct {
-	Host     string
-	Port     string
-	Database int
-	Username valuable.Valuable
-	Password valuable.Valuable
-	Key      string
+	Host     string            `json:"host"`
+	Port     string            `json:"port"`
+	Database int               `json:"database"`
+	Username valuable.Valuable `json:"username"`
+	Password valuable.Valuable `json:"password"`
+	Key      string            `json:"key"`
 }
 
 // NewStorage is the function for create new Redis storage client

tests/simple_template.tpl

@@ -0,0 +1 @@
+{{ .Request.Method }}

tests/template.tpl

@@ -0,0 +1,10 @@
+{
+  "config": "{{ toJson .Config }}",
+  "storage": {{ toJson .Storage }},
+  "metadata": {
+    "model": "{{ .Request.Header | getHeader "X-Model" }}",
+    "event": "{{ .Request.Header | getHeader "X-Event" }}",
+    "deliveryID": "{{ .Request.Header | getHeader "X-Delivery" | default "unknown" }}"
+  },
+  "payload": {{ .Payload }}
+}

tests/webhooks.tests.yml

@@ -4,6 +4,8 @@ observability:
 specs:
 - name: exampleHook
   entrypointUrl: /webhooks/example
+  formatting:
+    templateString: "{{ .Payload }}"
   security:
   - header:
       inputs: