Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Added SSL benchmark #16

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Added SSL benchmark #16

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
FROM ubuntu:16.04
FROM python:3.7
MAINTAINER elvis@magic.io

RUN DEBIAN_FRONTEND=noninteractive \
apt-get update && apt-get install -y \
language-pack-en
locales locales-all

ENV LANG en_US.UTF-8
ENV WORKON_HOME /usr/local/python-venvs
Expand All @@ -16,15 +16,16 @@ ENV GOPATH /usr/go/
RUN DEBIAN_FRONTEND=noninteractive \
apt-get update && apt-get install -y \
autoconf automake libtool build-essential \
python3 python3-pip git nodejs golang gosu
python3 python3-pip git nodejs golang gosu libssl-dev

RUN pip3 install vex
RUN vex --python=python3.5 -m bench pip install -U pip
RUN vex --python=python3 -m bench pip install -U pip
RUN mkdir -p /var/lib/cache/pip

ADD servers /usr/src/servers
RUN cd /usr/src/servers && go build goecho.go && \
go get github.com/golang/groupcache/lru && go build gohttp.go
go get github.com/golang/groupcache/lru && go build gohttp.go && \
g++ -g -Wall -Werror -O2 sslbench.cc -lssl -lcrypto -ldl -lpthread -o sslbench
RUN vex bench pip --cache-dir=/var/lib/cache/pip \
install -r /usr/src/servers/requirements.txt

Expand Down
24 changes: 20 additions & 4 deletions echo_client
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ if __name__ == '__main__':
help='server address')
parser.add_argument('--output-format', default='text', type=str,
help='output format', choices=['text', 'json'])
parser.add_argument('--ssl', action='store_true', help='enable SSL')
args = parser.parse_args()

unix = False
Expand All @@ -78,6 +79,16 @@ if __name__ == '__main__':

sock.settimeout(timeout / 1000)
sock.connect(addr)
cipher = None
if args.ssl:
import ssl
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
ssl_context.set_ciphers('ECDHE-RSA-AES128-GCM-SHA256')
sock = ssl_context.wrap_socket(sock)
sock.do_handshake()
cipher = '%s %s %dbits' % sock.cipher()

n = 0
latency_stats = np.zeros((timeout * 100,))
Expand Down Expand Up @@ -106,7 +117,7 @@ if __name__ == '__main__':
except OSError:
pass

return n, latency_stats, min_latency, max_latency
return n, latency_stats, min_latency, max_latency, cipher

N = args.concurrency
DURATION = args.duration
Expand All @@ -116,6 +127,7 @@ if __name__ == '__main__':
messages = 0
latency_stats = None
start = time.monotonic()
ciphers = set()

with futures.ProcessPoolExecutor(max_workers=N) as e:
fs = []
Expand All @@ -124,8 +136,9 @@ if __name__ == '__main__':

res = futures.wait(fs)
for fut in res.done:
t_messages, t_latency_stats, t_min_latency, t_max_latency = \
t_messages, t_latency_stats, t_min_latency, t_max_latency, cip = \
fut.result()
ciphers.add(cip)
messages += t_messages
if latency_stats is None:
latency_stats = t_latency_stats
Expand Down Expand Up @@ -167,7 +180,8 @@ if __name__ == '__main__':
latency_max=round(max_latency / 100, 3),
latency_std=round(latency_std / 100, 3),
latency_cv=round(latency_cv * 100, 2),
latency_percentiles=percentile_data
latency_percentiles=percentile_data,
ciphers=', '.join(ciphers),
)

if args.output_format == 'json':
Expand All @@ -183,7 +197,8 @@ if __name__ == '__main__':
"latency_max": {latency_max},
"latency_std": {latency_std},
"latency_cv": {latency_cv},
"latency_percentiles": {latency_percentiles}
"latency_percentiles": {latency_percentiles},
"ciphers": "{ciphers}"
}}'''.format(**data)
else:
data['latency_percentiles'] = '; '.join(
Expand All @@ -196,6 +211,7 @@ std: {latency_std}ms ({latency_cv}%)
Latency distribution: {latency_percentiles}
Requests/sec: {rps}
Transfer/sec: {transfer}MiB
Ciphers: {ciphers}
'''.format(duration=DURATION, size=round(MSGSIZE / 1024, 2), **data)

print(output)
121 changes: 120 additions & 1 deletion run_benchmarks
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ server_base = ['docker', 'run', '--rm', '-t', '-p', '25000:25000',
'-e', 'GID={}'.format(os.getegid()),
'-v', '{_cache}:/var/lib/cache'.format(_cache=_cache),
'-v', '{_socket}:/tmp/sockets'.format(_socket=_socket),
'-w', '/usr/src/servers',
'--name', 'magicbench', 'magic/benchmark']

python = ['vex', 'bench', 'python']
Expand All @@ -41,6 +42,7 @@ unix_client = echo_client + ['--addr={}'.format(unix_address)]
http_client = ['./http_client', '--output-format=json',
'--addr={}'.format(tcp_address)]
readline_client = tcp_client + ['--mpr=5']
ssl_client = tcp_client + ['--ssl']

benchmarks = [{
'name': 'tcpecho-gevent-sockets',
Expand Down Expand Up @@ -213,6 +215,60 @@ benchmarks = [{
'server': ['/usr/src/servers/gohttp'],
'server_address': tcp_address,
'client': http_client,
}, {
'name': 'ssl-threaded',
'title': 'SSL echo server (threaded)',
'server': python + ['/usr/src/servers/threadsslecho.py'],
'server_address': tcp_address,
'client': ssl_client,
}, {
'name': 'ssl-twisted',
'title': 'SSL echo server (twisted)',
'server': python + ['/usr/src/servers/twistedecho.py', '--ssl'],
'server_address': tcp_address,
'client': ssl_client,
}, {
'name': 'ssl-gevent',
'title': 'SSL echo server (gevent)',
'server': python + ['/usr/src/servers/gevecho.py', '--ssl'],
'server_address': tcp_address,
'client': ssl_client,
}, {
'name': 'ssl-golang-sockets',
'title': 'SSL echo server (golang)',
'server': ['/usr/src/servers/goecho', '--ssl'],
'server_address': tcp_address,
'client': ssl_client,
}, {
'name': 'ssl-nodejs-streams',
'title': 'SSL echo server (nodejs)',
'server': nodejs + ['/usr/src/servers/nodeecho.js', '--ssl'],
'server_address': tcp_address,
'client': ssl_client,
}, {
'name': 'ssl-asyncio-protocol',
'title': 'SSL echo server (asyncio)',
'server': python + ['/usr/src/servers/asyncioecho.py',
'--addr=0.0.0.0:25000', '--ssl',
'--proto'],
'server_address': tcp_address,
'client': ssl_client,
}, {
'name': 'ssl-uvloop-protocol',
'title': 'SSL echo server (uvloop)',
'server': python + ['/usr/src/servers/asyncioecho.py',
'--addr=0.0.0.0:25000', '--ssl',
'--proto', '--uvloop'],
'server_address': tcp_address,
'client': ssl_client,
}, {
'name': 'ssl-uvloop-buffered-protocol',
'title': 'SSL echo server (uvloop buffered)',
'server': python + ['/usr/src/servers/asyncioecho.py',
'--addr=0.0.0.0:25000', '--ssl',
'--proto', '--uvloop', '--buffered'],
'server_address': tcp_address,
'client': ssl_client,
}]


Expand Down Expand Up @@ -245,6 +301,13 @@ def start_and_wait_for_server(server_cmd, address, timeout=60):
sock.settimeout(time.monotonic() - start)
try:
sock.connect(addr)
if 'ssl' in (' '.join(server_cmd)).lower():
import ssl
ssl_context = ssl.create_default_context()
ssl_context.check_hostname = False
ssl_context.verify_mode = ssl.CERT_NONE
sock = ssl_context.wrap_socket(sock)
sock.do_handshake()
sock.sendall(b'GET / HTTP/1.0\r\n\r\n')
if sock.recv(4):
print('Server is up and running.')
Expand Down Expand Up @@ -297,7 +360,7 @@ def kill_server():

if server_container_exists():
print('Removing server container...')
subprocess.check_output(['docker', 'rm', 'magicbench'])
subprocess.check_output(['docker', 'rm', '-f', 'magicbench'])


def format_report(data, target_file):
Expand Down Expand Up @@ -348,6 +411,7 @@ def format_report(data, target_file):
('Mean latency', []),
('Max latency', []),
('Latency variation', []),
('Ciphers', [])
))

variations = benchmark['variations']
Expand All @@ -370,6 +434,7 @@ def format_report(data, target_file):
'{}ms'.format(variation['latency_max']))
brecords['Latency variation'].append('{}ms ({}%)'.format(
variation['latency_std'], variation['latency_cv']))
brecords['Ciphers'].append(variation['ciphers'])

vc = len(data['concurrency_levels']) * len(data['payload_size_levels'])

Expand Down Expand Up @@ -538,6 +603,7 @@ def main():
Latency distribution: {latency_percentiles}
Requests/sec: {rps}
Transfer/sec: {transfer}MiB
Ciphers: {ciphers}
''').format(duration=duration, **format_data)

print(output)
Expand All @@ -548,6 +614,59 @@ def main():

print()

if 'ssl' in args.benchmarks:
title = 'ssl-raw'
print(title)
print('=' * len(title))
print()

benchmark_data = {
'name': title,
'variations': []
}

benchmarks_data.append(benchmark_data)
ssl_cmd = server_base + ['/usr/src/servers/sslbench', 'bulk',
'ECDHE-RSA-AES128-GCM-SHA256']
for variation in variations:
title = 'BENCHMARK: {}'.format(variation['title'])
print(title)
print('-' * len(title))
msgsize = variation['payload_size']

cmd = ssl_cmd + [str(msgsize)]
print(' ' + ' '.join(cmd))
output = subprocess.check_output(cmd, universal_newlines=True)
data = json.loads(output)
data['transfer'] = round(msgsize * data['messages'] /
data['duration'] / 1024 / 1024, 2)
data['rps'] = round(data['messages'] / data['duration'], 2)
avg = round(data['duration'] / data['messages'], 3)
data['latency_mean'] = data['latency_std'] = data['latency_cv'] = avg
data['latency_min'] = round(data['latency_min'], 3)
data['latency_max'] = round(data['latency_max'], 3)
data['latency_percentiles'] = [(p, avg) for p in [25, 50, 75, 90, 99, 99.99]]

format_data = data.copy()

format_data['latency_percentiles'] = '; '.join(
'{}% under {}ms'.format(*v)
for v in data['latency_percentiles'])

output = textwrap.dedent('''\
{messages} messages in {duration} seconds
Latency: min {latency_min}ms; max {latency_max}ms; mean {latency_mean}ms; std {latency_std}ms ({latency_cv}%);
Latency distribution: {latency_percentiles}
Requests/sec: {rps}
Transfer/sec: {transfer}MiB
Ciphers: {ciphers}
''').format(**format_data)

print(output)
benchmark_data['variations'].append(data)

print()

if args.save_json or args.save_html:
info_cmd = server_base + python + ['/usr/src/servers/platinfo.py']
print(' ' + ' '.join(info_cmd))
Expand Down
Loading