fabiocicerchia · fabiocicerchia · Sep 17, 2021 · Sep 16, 2021 · Sep 16, 2021 · Sep 16, 2021
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -78,10 +78,10 @@ jobs:
       - name: Test
         run: |
           echo "127.0.0.1 testing.local www.testing.local" | tee -a /etc/hosts
-          cd test/full-setup
+          cd test/full-setup/certs
           ./gen-selfsigned-cert.sh
           ./gen-selfsigned-cert.sh www.w3.org
-          cd ../..
+          cd ../../..
           ./go-proxy-cache -debug -config=test/full-setup/config.yml &
           make test
         env:

diff --git a/.gitignore b/.gitignore
@@ -49,7 +49,7 @@
 __debug_bin
 coverage.txt
 dump.rdb
-test/full-setup/node_modules
+test/full-setup/ws/node_modules
 test/full-setup/certs/*/*.pem
 test/full-setup/certs/*/*.srl
 test/full-setup/certs/*/*.csr

diff --git a/Makefile b/Makefile
@@ -100,8 +100,8 @@ test-endtoend: ## test endtoend
 	go test -v -race -count=1 --tags=endtoend ./...
 
 test-ws: ## test websocket
-	cd test/full-setup && npm install
-	node test/full-setup/ws_client.js
+	cd test/full-setup/ws && npm install
+	node test/full-setup/ws/ws_client.js
 
 test-http2: ## test HTTP2
 	MATCHING=$(shell nghttp -ans https://testing.local:50443/push 2> /dev/null | grep 200 | wc -l | xargs); \

diff --git a/README.md b/README.md
@@ -104,7 +104,7 @@ you --->|---->----|--->---|---'     |       |     |   |
 
 ### Reliability
 
-- **Healthcheck Endpoint**, exposes the route `/healthcheck` (optional).
+- **Healthcheck Endpoint**, exposes the route `/healthcheck` (internally).
 - **Upstream Healthcheck**, verifies periodically if upstream nodes are healthy.
 - **Respecting HTTP Cache Headers**, `Vary`, `ETag`, `Cache-Control` and `Expires`.
 - **Fully Tested**, Unit, Functional & Linted & 0 Race Conditions Detected.
@@ -127,6 +127,7 @@ you --->|---->----|--->---|---'     |       |     |   |
 
 - **Request Tracing**, each line in logs has a RequestID to easily identify the response flow.
 - **OpenTelemetry Tracing**, each request has a deep tracing with Jaeger (optional).
+- **Prometheus Endpoint**, exposes the route `/metrics` (internally) to serve Prometheus metrics.
 
 ## Configuration
 

diff --git a/cache/engine/client/client.go b/cache/engine/client/client.go
@@ -73,7 +73,7 @@ func (rdb *RedisClient) getMutex(key string) *redsync.Mutex {
 
 func (rdb *RedisClient) lock(key string) error {
 	if err := rdb.getMutex(key).Lock(); err != nil {
-		rdb.logger.Errorf("Lock Error on %s: %s", key, err) // TODO: Add to trace span?
+		rdb.logger.Errorf("Lock Error on %s: %s", key, err) // TODO! Add to trace span?
 		return err
 	}
 
@@ -82,7 +82,7 @@ func (rdb *RedisClient) lock(key string) error {
 
 func (rdb *RedisClient) unlock(key string) error {
 	if ok, err := rdb.getMutex(key).Unlock(); !ok || err != nil {
-		rdb.logger.Errorf("Unlock Error on %s: %s", key, err) // TODO: Add to trace span?
+		rdb.logger.Errorf("Unlock Error on %s: %s", key, err) // TODO! Add to trace span?
 		return err
 	}
 

diff --git a/config.yml.dist b/config.yml.dist
@@ -268,7 +268,7 @@ log:
 
 # --- TRACING
 tracing:
-  # Endpoint for Jaeger (eg: http://jaeger:14268/api/traces)
+  # Endpoint for Local Jaeger Agent (eg: jaeger:6831)
   jaeger_endpoint: ~
   # Enable/Disable the tracing.
   enabled: false

diff --git a/config/model.go b/config/model.go
@@ -64,12 +64,11 @@ type Domains map[string]Configuration
 
 // Server - Defines basic info for the server.
 type Server struct {
-	Port        Port     `yaml:"port"`
-	TLS         TLS      `yaml:"tls"`
-	Timeout     Timeout  `yaml:"timeout"`
-	Upstream    Upstream `yaml:"upstream"`
-	GZip        bool     `yaml:"gzip" envconfig:"GZIP_ENABLED"`
-	Healthcheck bool     `yaml:"healthcheck"`
+	Port     Port     `yaml:"port"`
+	TLS      TLS      `yaml:"tls"`
+	Timeout  Timeout  `yaml:"timeout"`
+	Upstream Upstream `yaml:"upstream"`
+	GZip     bool     `yaml:"gzip" envconfig:"GZIP_ENABLED"`
 }
 
 // Port - Defines the listening ports per protocol.
@@ -204,8 +203,7 @@ var Config Configuration = Configuration{
 				Scheme:      "https",
 			},
 		},
-		GZip:        false,
-		Healthcheck: true,
+		GZip: false,
 	},
 	Cache: Cache{
 		Port:            "6379",

diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md
@@ -312,7 +312,7 @@ log:
 
 # --- TRACING
 tracing:
-  # Endpoint for Jaeger (eg: http://jaeger:14268/api/traces)
+  # Endpoint for Local Jaeger Agent (eg: jaeger:6831)
   jaeger_endpoint: ~
   # Enable/Disable the tracing.
   enabled: false

diff --git a/docs/DEV.md b/docs/DEV.md
@@ -20,7 +20,16 @@ $ docker-compose up
 
 ## Tracing
 
-Jaeger is available by using the `test/full-setup` demo and browsing to `http://localhost:16686/`.
+Jaeger is available by using the `test/full-setup` demo and browsing to `http://127.0.0.1:16686/`.
+
+Prometheus endpoint is available at `http://127.0.0.1:52021/metrics`. Its metrics are collected and available via Grafana
+at `http://localhost:3001`.
+
+There is a JSON export of the dashboard stored in `test/full-setup/grafana/gpc-dashboard.json`.
+
+![GPC Grafana Dashboard](grafana.png)
+
+Note: the Data Source must be configured in Grafana to point to `http://prometheus:9090`.
 
 ## Test
 

diff --git a/docs/EXAMPLES.md b/docs/EXAMPLES.md
@@ -1,5 +1,17 @@
 # Examples
 
+## Playground
+
+In the folder `test/full-setup` there is complete working environment in Docker:
+
+- GoProxyCache
+- Redis
+- Nginx (for upstream)
+- NodeJS (for ws/wss server)
+- Jaeger (for OpenTracing)
+- Prometheus (for pulling metrics)
+- Grafana (for visualising metrics)
+
 ## CLI
 
 ```console

diff --git a/docs/KUBERNETES.md b/docs/KUBERNETES.md
@@ -1,7 +1,7 @@
 # Kubernets
 
 ```console
-$ kubectl apply -f test/full-setup/k8s.yml
+$ kubectl apply -f test/full-setup/kubernetes/k8s.yml
 service/go-proxy-cache created
 deployment.apps/go-proxy-cache created
 service/nginx created

diff --git a/docs/METRICS.md b/docs/METRICS.md
@@ -0,0 +1,65 @@
+# Metrics
+
+Go Proxy Cache supports application metrics via Prometheus endpoint, `/metrics`.
+
+For every request these are the metrics collected:
+
+- `gpc_status_codes_total`  
+   Distribution by status codes  
+   Labels: hostname, env, code
+
+- `gpc_request_host_total`  
+   Distribution by Request Host  
+   Labels: hostname, env, host
+
+- `gpc_http_methods_total`  
+   Distribution by HTTP methods  
+   Labels: hostname, env, method
+
+- `gpc_url_scheme_total`  
+   Distribution by URL scheme  
+   Labels: hostname, env, scheme
+
+- `gpc_request_sum_total`  
+   Total number of sent requests  
+   Labels: hostname, env
+
+- `gpc_request_1xx_total`  
+   Total number of sent 1xx requests  
+   Labels: hostname, env
+
+- `gpc_request_2xx_total`  
+   Total number of sent 2xx requests  
+   Labels: hostname, env
+
+- `gpc_request_3xx_total`  
+   Total number of sent 3xx requests  
+   Labels: hostname, env
+
+- `gpc_request_4xx_total`  
+   Total number of sent 4xx requests  
+   Labels: hostname, env
+
+- `gpc_request_5xx_total`  
+   Total number of sent 5xx requests  
+   Labels: hostname, env
+
+- `gpc_host_healthy`  
+   Health state of hosts  
+   Labels: hostname, env
+
+- `gpc_host_unhealthy`  
+   Health state of hosts  
+   Labels: hostname, env
+
+- `gpc_cache_hits_total`  
+   The amount of cache hits  
+   Labels: hostname, env
+
+- `gpc_cache_miss_total`  
+   The amount of cache misses  
+   Labels: hostname, env
+
+- `gpc_cache_stale_total`  
+   The amount of cache misses  
+   Labels: hostname, env
diff --git a/docs/REAL-CASE.md b/docs/REAL-CASE.md
@@ -233,7 +233,7 @@ server:
 
 # --- TRACING
 tracing:
-  # Endpoint for Jaeger (eg: http://jaeger:14268/api/traces)
+  # Endpoint for Local Jaeger Agent (eg: jaeger:6831)
   jaeger_endpoint: ~
   # Enable/Disable the tracing.
   enabled: false

diff --git a/docs/TRACING.md b/docs/TRACING.md
@@ -1,11 +1,11 @@
 # Tracing
 
-Go Proxy Cache supports OpenTelemetry, specifically Jaeger.
+Go Proxy Cache supports OpenTracing, specifically Jaeger.
 
 Use the environment variable `TRACING_ENV` to customise the tracing.
 
 There is a unique key used in order to be able to match the request against multiple services.
-The key in OpenTelemetry is `request.id`, it is also sent to the upstream backend as an additional HTTP header (ie. `X-Go-Proxy-Cache-Request-ID`).
+The main Request ID key in OpenTracing is `request.id`, it is also sent to the upstream backend as an additional HTTP header (ie. `X-Go-Proxy-Cache-Request-ID`).
 
 For every request these are the tracing spans (and relative tags) created:
 

diff --git a/docs/grafana.png b/docs/grafana.png
diff --git a/go.mod b/go.mod
@@ -3,6 +3,7 @@ module github.com/fabiocicerchia/go-proxy-cache
 go 1.14
 
 require (
+	github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
 	github.com/go-http-utils/fresh v0.0.0-20161124030543-7231e26a4b27
 	github.com/go-http-utils/headers v0.0.0-20181008091004-fed159eddc2a
 	github.com/go-redis/redis/v8 v8.11.3
@@ -12,20 +13,18 @@ require (
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/pkg/errors v0.9.1
+	github.com/prometheus/client_golang v1.11.0
 	github.com/rs/dnscache v0.0.0-20210201191234-295bba877686
 	github.com/rs/xid v1.3.0
 	github.com/sirupsen/logrus v1.8.1
 	github.com/sony/gobreaker v0.4.1
-	github.com/stretchr/objx v0.1.1 // indirect
 	github.com/stretchr/testify v1.7.0
 	github.com/uber/jaeger-client-go v2.29.1+incompatible
 	github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
 	github.com/ugorji/go/codec v1.2.6
 	github.com/yhat/wsutil v0.0.0-20170731153501-1d66fa95c997
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.23.0
 	go.opentelemetry.io/otel v1.0.0-RC3.0.20210914151502-ec26b556da5c
-	go.opentelemetry.io/otel/exporters/jaeger v1.0.0-RC3.0.20210914151502-ec26b556da5c // indirect
-	go.opentelemetry.io/otel/sdk v1.0.0-RC3 // indirect
 	go.opentelemetry.io/otel/trace v1.0.0-RC3
 	go.uber.org/atomic v1.9.0 // indirect
 	golang.org/x/crypto v0.0.0-20210817164053-32db794688a5