From 522177e925a1162b2cb4ab710868df04800498d8 Mon Sep 17 00:00:00 2001 From: Carlo Cabanilla Date: Thu, 27 Mar 2014 17:24:07 -0400 Subject: [PATCH 1/2] compress dogstatsd payloads --- dogstatsd.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dogstatsd.py b/dogstatsd.py index a91c8690f3..e99aae8797 100755 --- a/dogstatsd.py +++ b/dogstatsd.py @@ -19,6 +19,7 @@ import signal import socket import sys +import zlib from time import time import threading from urllib import urlencode @@ -43,7 +44,7 @@ EVENT_CHUNK_SIZE = 50 def serialize_metrics(metrics): - return json.dumps({"series" : metrics}) + return zlib.compress(json.dumps({"series" : metrics})) def serialize_event(event): return json.dumps(event) @@ -147,7 +148,8 @@ def submit(self, metrics): # Copy and pasted from dogapi, because it's a bit of a pain to distribute python # dependencies with the agent. body = serialize_metrics(metrics) - headers = {'Content-Type':'application/json'} + headers = {'Content-Type': 'application/json', + 'Content-Encoding': 'deflate'} method = 'POST' params = {} From 15b02fba14afbff289455ffd8c8b70ea253f376d Mon Sep 17 00:00:00 2001 From: Carlo Cabanilla Date: Wed, 2 Apr 2014 22:06:47 +0000 Subject: [PATCH 2/2] only compress dogstatsd payloads that are larger than 1k --- dogstatsd.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/dogstatsd.py b/dogstatsd.py index e99aae8797..d5e4be3d18 100755 --- a/dogstatsd.py +++ b/dogstatsd.py @@ -42,9 +42,17 @@ FLUSH_LOGGING_INITIAL = 10 FLUSH_LOGGING_COUNT = 5 EVENT_CHUNK_SIZE = 50 +COMPRESS_THRESHOLD = 1024 def serialize_metrics(metrics): - return zlib.compress(json.dumps({"series" : metrics})) + serialized = json.dumps({"series" : metrics}) + if len(serialized) > COMPRESS_THRESHOLD: + headers = {'Content-Type': 'application/json', + 'Content-Encoding': 'deflate'} + serialized = zlib.compress(serialized) + else: + headers = {'Content-Type': 'application/json'} + return serialized, headers def serialize_event(event): return json.dumps(event) @@ -147,9 +155,7 @@ def flush(self): def submit(self, metrics): # Copy and pasted from dogapi, because it's a bit of a pain to distribute python # dependencies with the agent. - body = serialize_metrics(metrics) - headers = {'Content-Type': 'application/json', - 'Content-Encoding': 'deflate'} + body, headers = serialize_metrics(metrics) method = 'POST' params = {}